def main():
network_dataset = Dataset('twitters2')
nl = read_file_to_dict(os.path.join(DATASET_PATH, 'TwitterSample2.txt'))
# 10% sampling
nbunch = nl[0:int(len(nl) // 2)]
network_dataset.graph = network_dataset.graph.subgraph(nbunch)
server_list = [Server(k) for k in range(0, 512)]
vp_number = 0
node_list = list(network_dataset.graph.nodes)
random.shuffle(node_list)
print('Dataset information: TwitterSample2\nNodes Number:',
network_dataset.graph.order(), '\nEdge Number:',
network_dataset.graph.size())
print('Using Random Partitioning Method...\nServer Number:',
len(server_list), '\nVirtual Primary Copy Number:', vp_number,
'\nWrite Frequency of Nodes: 1')
start = time.time()
m = RandomP(server_list, network_dataset, node_list)
m.add_new_primary_node(server_list, vp_number)
m.check_server_load()
m.check_locality()
end = time.time()
print('Random Partitioning Time:', end - start, 'seconds')
m.compute_inter_sever_cost()
path = RANDOM_GRAPH_PATH
m.save_all(path)
def create_algo(server_count=4, node_count=10):
data = Dataset(dataset_str='facebook')
data.graph = nx.Graph()
for i in range(node_count):
data.graph.add_node(i)
server_list = [Server(serer_id=i) for i in range(server_count)]
algo = OfflineAlgo(server_list=server_list, network_dataset=data)
return algo
def test_relocate_process(self):
data = Dataset(dataset_str='facebook')
data.graph = nx.Graph()
for i in range(10):
data.graph.add_node(i)
data.graph.add_edge(0, 1)
data.graph.add_edge(0, 2)
data.graph.add_edge(0, 3)
data.graph.add_edge(0, 4)
server_list = [Server(serer_id=i) for i in range(8)]
algo = OfflineAlgo(server_list=server_list, network_dataset=data)
node_list = list(data.graph.nodes)
node_len = len(node_list)
for i in range(node_len):
n = node_list[i]
algo.add_new_primary_node(node_id=n, write_freq=Constant.WRITE_FREQ)
algo.node_relocation_process()
def test_merge_process(self):
data = Dataset(dataset_str='facebook')
data.graph = nx.Graph()
for i in range(10):
data.graph.add_node(i)
data.graph.add_edge(0, 1)
data.graph.add_edge(0, 2)
data.graph.add_edge(0, 3)
data.graph.add_edge(0, 4)
server_list = [Server(serer_id=i) for i in range(8)]
algo = OfflineAlgo(server_list=server_list, network_dataset=data)
node_list = list(data.graph.nodes)
node_len = len(node_list)
for i in range(node_len):
n = node_list[i]
algo.add_new_primary_node(node_id=n, write_freq=Constant.WRITE_FREQ)
algo.init_merge_process()
for i in range(0, len(algo.merged_node_list)):
m_node = algo.merged_node_list[i]
if m_node.id == 0:
self.assertEqual(m_node.internal_connection, 0)
self.assertEqual(m_node.external_connection, 4)
elif m_node.id in [1, 2, 3, 4]:
self.assertEqual(m_node.internal_connection, 0)
self.assertEqual(m_node.external_connection, 1)
else:
self.assertEqual(m_node.internal_connection, 0)
self.assertEqual(m_node.external_connection, 0)
node_count_list = []
for m_node in algo.merged_node_list:
node_count_list += m_node.node_id_list
node_count_list.sort()
self.assertEqual(node_count_list, [i for i in range(10)])
for i in range(1, len(algo.merged_node_list)):
algo.merged_node_list[0]._add_node(algo.merged_node_list[i], algo=algo, remove_flag=False)
node_count_list = algo.merged_node_list[0].node_id_list
node_count_list.sort()
self.assertEqual(node_count_list, [i for i in range(10)])
self.assertEqual(algo.merged_node_list[0].external_connection, 0)
self.assertEqual(algo.merged_node_list[0].internal_connection, 4)
self.assertEqual(algo.merged_node_list[0].node_count, 10)
def test_virtual_primary_copy_process(self):
data = Dataset(dataset_str='facebook')
data.graph = nx.Graph()
data.graph.add_node(0)
server_list = [Server(serer_id=i) for i in range(2)]
Constant.LEAST_VIRTUAL_PRIMARY_COPY_NUMBER = 1
algo = OfflineAlgo(server_list=server_list, network_dataset=data)
node_list = list(data.graph.nodes)
node_len = len(node_list)
for i in range(node_len):
n = node_list[i]
algo.add_new_primary_node(node_id=n, write_freq=Constant.WRITE_FREQ)
self.assertEqual(len(algo.node_list), 1)
self.assertEqual(algo.node_list[0].id, 0)
self.assertEqual(algo.node_list[0].virtual_primary_copy_server_list[0].id, 1 - algo.node_list[0].server.id)
self.assertEqual(len(algo.node_list[0].virtual_primary_copy_server_list), 1)
data.graph.add_edge(0, 1)
algo.server_list.append(Server(2))
algo.server_list.append(Server(3))
algo._add_node_to_server(node_id=1, node_type=Constant.PRIMARY_COPY, write_freq=10.0,
server=algo.server_list[2])
# algo.virtual_primary_copy_swap()
for vir_server in algo.node_list[0].virtual_primary_copy_server_list:
if vir_server.id != algo.node_list[1].virtual_primary_copy_server_list[0].id:
tmp_server_1_id = vir_server
tmp_server_2_id = algo.node_list[1].virtual_primary_copy_server_list[0]
Operation.swap_virtual_primary_copy(s_node=algo.node_list[0],
t_node=algo.node_list[1],
s_server=vir_server,
t_server=algo.node_list[1].virtual_primary_copy_server_list[0],
algo=algo)
break
self.assertTrue(tmp_server_1_id.has_node(algo.node_list[1].id, node_type=Constant.VIRTUAL_PRIMARY_COPY))
self.assertTrue(tmp_server_2_id.has_node(algo.node_list[0].id, node_type=Constant.VIRTUAL_PRIMARY_COPY))
self.assertTrue(tmp_server_2_id in algo.node_list[0].virtual_primary_copy_server_list)
self.assertTrue(tmp_server_1_id in algo.node_list[1].virtual_primary_copy_server_list)
def test_merged_node_swap_process(self):
data = Dataset(dataset_str='facebook')
data.graph = nx.Graph()
for i in range(10):
data.graph.add_node(i)
data.graph.add_edge(0, 1)
data.graph.add_edge(0, 2)
data.graph.add_edge(0, 3)
data.graph.add_edge(0, 4)
server_list = [Server(serer_id=i) for i in range(8)]
algo = OfflineAlgo(server_list=server_list, network_dataset=data)
node_list = list(data.graph.nodes)
node_len = len(node_list)
for i in range(node_len):
n = node_list[i]
algo.add_new_primary_node(node_id=n, write_freq=Constant.WRITE_FREQ)
algo.init_merge_process()
algo.start_merge_process()
node_count_list = []
for m_node in algo.merged_node_list:
node_count_list += m_node.node_id_list
node_count_list.sort()
self.assertEqual(node_count_list, [i for i in range(10)])
