/
split_xform.py
executable file
·93 lines (80 loc) · 2.58 KB
/
split_xform.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
#!/usr/bin/env python
""" This module demonstrates some features of the libstreampy
We take an iterable and split it into even and odd then scan it
Comparing a static implementation to the stream.py implementation
Depends on the stream module from https://github.com/aht/stream.py.
"""
from __future__ import print_function
import sys
sys.path.append("~/Code/streaming/libstreampy/")
from time import time
import math
import stream
from stream import fold
from stream import tee
from stream import filter
from stream import item
import operator as ops
if __name__ == '__main__':
if len(sys.argv) != 2:
print("Usage: ./split_xform.py N", file=sys.stderr)
sys.exit(1)
N = int(sys.argv[1])
instream = [float(i) for i in range(N)]
def scan(op, seq):
"""Compute the op-scan of seq"""
l = []
s = 0
for x in seq:
s = op(s,x)
l.append(s)
return l
def my_sqrt(x):
for i in x:
yield math.sqrt(i)
#static computation in a time block
ts = time()
evens = instream[::2]
odds = instream[1::2]
evens = map(math.sqrt, evens)
odds = map(math.sqrt, odds)
even_ans = scan(ops.add, evens)
odd_ans = reduce(ops.add, odds )
static_time = time() - ts
#streaming computation
# create our filters
cong_2 = lambda x: x%2==0
evens = filter(cong_2)
odds = filter(lambda x: not cong_2(x))
ts = time()
# wire the split into the filters
instream >> tee(evens)
instream >> odds
# wire up the map and fold (scan/accumulate)
foldedevens = (evens >> stream.map(math.sqrt) >> fold(ops.add))
print(time() - ts)
sqrtodds = odds >> (stream.Processor(my_sqrt))
print("established the sqrter %f" % (time() - ts))
foldedodd = sqrtodds >> stream.fold(ops.add)
print("made odd folder: %f" % (time() - ts))
# force execution
soans = foldedodd >> item[-1:]
print(soans)
print(time() - ts)
seans = foldedevens >> item[:]
print(time() - ts)
stream_time = time() - ts
#print(even_ans)
#print(seans >> item[:])
#print(odd_ans)
#print(soans >> item[:])
#check the answers
#assert ( seans == (even_ans))
#assert ( soans == (odd_ans))
assert ( seans == list(even_ans))
#assert ( soans == list(odd_ans))
# display the timing information
print("we got the same answer using the streams")
print("static_time: %f, ips: %f" % (static_time, N//static_time))
print("stream_time: %f, ips: %f" % (stream_time, N//stream_time))
print("streaming/static: %f" % (stream_time/static_time))