/
compute_local_optimized_step_4.py
executable file
·161 lines (120 loc) · 5.44 KB
/
compute_local_optimized_step_4.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
from multiprocessing import Process, Manager
import time
from map import map
from combine import combine
from reduce import reduce
from client import get_num_map_nodes, create_list
"""
Notice
this module should not be altered unless one wishes to:
* test the algorithm at some fixed number of map nodes
uncomment line that contains compute(lst,n,'test')
* test run time at different input sizes
uncomment the for-loop block under __name__ == '__main__'
All neccessary parameters is customizable in client.py
client must provide in client.py the following:
* a method that generates a list using create_lst(), empty list is okay
* a method that specifies the number of map nodes
defined as get_num_map_nodes(size_param)
and size_param is an input size parameter the client can use
to adjust number of map nodes
* a weight method, w(input), that takes the element
of the given list and returns an integer or float
see (map.py) to see how the weight method is used
* a substitute method, x(), which returns an integer
or float to replace the variable x in the series
About the Module
compute(lst) computes the sum of the generating series
for a set of elements (lst) implemented as a Python list with respect
to weight function w() (to be defined in client.py).
optional paramers:
size_param used to determine appropriate number of map nodes
option 'not_fixed' or 'fixed'
where
'fixed' tells compute() to
keep number of map nodes at some default values
to be defined in the client module
'not_fixed' will
lst may be a list of any type (integer, list, etc), as long as
it is consistent in the implementation of w()
User may also provide a parameter for n which can be
used to set number of map nodes
"""
def compute(lst, size_param = 0, option='not_fixed'):
try:
# number of map nodes num_map_nodes
if option=='not_fixed':
num_map_nodes = get_num_map_nodes(size_param)
elif option == 'fixed':
num_map_nodes = 5
else:
raise NameError('second parameter [option] should be either `not_fixed` or `fixed` ... see documentation')
# Manager to keep track of all map results
list_of_emissions = []
# Job is a list of processes
jobs = []
# Process number
num_p = 0
manager = Manager()
start_time = time.time()
print ("Up to Permutation:", str(time.time() - start_time ) + " seconds" )
len_lst = len(lst)
print ("Size of S: ", len_lst)
len_sublist = max(1,int(len_lst/float(num_map_nodes)))
i=0
#Split input into muliple sliced_list each sent to one map node
while i < len_lst:
list_of_emissions.append(manager.list())
if (len_lst - i <= len_sublist):
# last step
sliced_list = lst[i:]
else:
sliced_list = lst[i:i + len_sublist]
#print(len(sliced_list))
p = Process(target=map, args=(num_p, sliced_list, list_of_emissions[-1]))
i += len_sublist
jobs.append(p)
p.start()
num_p += 1
print ("Wait for Catch Up\n")
for p in jobs:
p.join()
print ("Up to Mapping Stage:", str(time.time() - start_time ) + " seconds" )
#---------------------------------------------------
# Combine Step
jobs = []
manager_3 = Manager()
combined_lists = []
for emissions in list_of_emissions:
for key in range(emissions[-1][1],emissions[-1][2]+5):
combined_lists.append(manager_3.list())
q = Process(target=combine, args=(key,emissions,combined_lists[-1]))
q.start()
print ("Wait for Catch Up\n")
for q in jobs:
q.join()
jobs = []
manager_2 = Manager()
result_lst = manager_2.list()
# REDUCE --------------------------------------------------------
for combined_list in combined_lists:
r = Process(target=reduce, args=(key,combined_list,result_lst))
jobs.append(r)
r.start()
print ("Wait for Catch Up\n")
for r in jobs:
r.join()
print ("Up to Reducing Stage:", str(time.time() - start_time ) + " seconds" )
print("Length of List of List: ", len_lst )
return sum(result_lst)
except NameError as e:
print('Usage Error:', e)
if __name__ == "__main__":
with open("output.txt", "a") as myfile:
# compute(create_list(n),,'fixed') # run test for fixed number of map nodes
for n in [4,11,12,13,14,15,16,17]: # run test for list of all subsets of {1...n}
start_time = time.time()
computed = str(compute(create_list(n), n))
myfile.write(str(n) + " "+ computed + " "+ str(time.time() - start_time) +"\n")
myfile.flush()
pass