forked from carinezhang/distributed-memory
/
master.py
154 lines (137 loc) · 4.81 KB
/
master.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
from mpi4py import MPI
import time
import sys
import numpy as np
from tags import Tags
from slave import Slave
class Master:
"""
The main process creates one or more of this class that handle groups of
slave processes
"""
def __init__(self, max_size):
self.comm = MPI.COMM_WORLD
self.status = MPI.Status()
self.slaves = set(range(1, MPI.COMM_WORLD.Get_size()))
self.max_size = max_size
def get_size(self):
'''
get_size() get a list of tuple with the number of the processus and its size
'''
lens = []
for i in range(1,self.comm.Get_size()):
self.comm.send(None, dest=i, tag=Tags.GET_SIZE)
length = self.comm.recv(source=i, tag=Tags.GET_SIZE)
lens.append((i,length))
return lens
def allocate(self, obj):
"""
Allocate the variable 'obj' and return an id associate with the variable.
Can handle 'int' and 'list' of 'int' types.
"""
lens = sorted(self.get_size(), key=lambda x: x[1])
size = lens[0][1]
if isinstance(obj, int):
send_size = 1
else:
send_size = len(obj)
#if we have enough space to stock everything at once
if self.max_size - size >= send_size:
p = lens[0][0]
size = lens[0][1]
self.comm.send((obj, time.time()), dest=p, tag=Tags.ALLOC)
var = self.comm.recv(source=p, tag=Tags.ALLOC)
return (type(obj), ['{}-{}-{}'.format(p,send_size, var)])
total_lens = sum(self.max_size - n for _, n in lens)
if total_lens < len(obj):
raise Exception('not enough space')
#when the list can't fit in one process
list_var = []
curr = 0
for p, length in lens:
disponible_size = self.max_size - length
up = curr + disponible_size
if up > len(obj):
up = len(obj)
send_size = up-curr
self.comm.send((obj[curr:up], time.time()), dest=p, tag=Tags.ALLOC)
list_var.append('{}-{}-{}'.format(p, send_size, self.comm.recv(source=p, tag=Tags.ALLOC)))
curr += disponible_size
if curr > len(obj):
break
return (list, list_var)
def read(self, var):
"""
Return the value associated with the var.
"""
var_list = var[1]
if len(var[1]) <= 1:
v = var_list[0].split('-')
p = int(v[0])
key = v[2]
self.comm.send(key, dest=p, tag=Tags.READ)
return self.comm.recv(source=p, tag=Tags.READ)
res = []
is_none = True
for v in var_list:
p = int(v.split('-')[0])
key = v.split('-')[2]
self.comm.send(key, dest=p, tag=Tags.READ)
tmp = (self.comm.recv(source=p, tag=Tags.READ))
if tmp != None:
is_none = False
res.extend(tmp)
if is_none:
return None
return res
def modify(self, var_name, new_val, index=None):
"""
Replace a variable by a new one.
Return True if the variable is modified, False otherwise.
"""
if var_name[0] == int:
p = int(var_name[1][0].split('-')[0])
key = var_name[1][0].split('-')[2]
self.comm.send((key, new_val, 0, time.time()), dest=p, tag=Tags.MODIFY)
return self.comm.recv(source=p, tag=Tags.MODIFY)
pos = 0
for v in var_name[1]:
tmp = v.split('-')
p = int(tmp[0])
size = int(tmp[1])
key = tmp[2]
if pos + size > index:
self.comm.send((key, new_val, index-pos, time.time()), dest=p, tag=Tags.MODIFY)
return self.comm.recv(source=p, tag=Tags.MODIFY)
pos += size
return False
def free(self, var):
"""
Free a variable.
Return True if variable has been erased, False otherwise.
"""
for v in var[1]:
tmp = v.split('-')
p = int(tmp[0])
size = int(tmp[1])
key = tmp[2]
self.comm.send((key, time.time()), dest=p, tag=Tags.FREE)
tmp = self.comm.recv(source=p, tag=Tags.FREE)
if tmp == False:
return False
return True
def terminate_slaves(self):
"""
Call this to make all slaves exit their run loop
"""
queue = []
for s in self.slaves:
queue.append(self.comm.isend(obj=None, dest=s, tag=Tags.EXIT))
for q in queue:
q.wait()
sys.exit(0)
def init(max_size=100):
if MPI.COMM_WORLD.Get_rank() == 0: # Master
return Master(max_size)
# Any slave
Slave().run()