def confirm_download_schedule(schedule):
"""Reports the total number of bytes and total number of files
to download. Also lists the inaccessible files (based on HEAD
response). Then asks user to confirm downloading.
"""
def content_length(tpl):
return tpl[2][1]
def status_code(tpl):
return tpl[2][0]
def href(tpl):
return tpl[0]
def is_OK(tpl):
return status_code(tpl) == 200
def not_OK(tpl):
return status_code(tpl) != 200
increment = lambda x, _: x + 1
file_count = (
schedule >> stream.filter(is_OK) >> stream.reduce(increment, 0))
bytes_to_download = (
schedule >> stream.filter(is_OK) >> stream.map(content_length) >> sum)
inaccessible_files = (schedule >> stream.filter(not_OK) >> list)
if len(inaccessible_files) > 0:
print
print "Some files are inaccessible:"
for (idx, sched) in enumerate(inaccessible_files):
print "%d: %d %s" % (idx, status_code(sched), href(sched))
if bytes_to_download > 0:
print
print "Need to download %s in %d files." % (
pretty_bytes(bytes_to_download), file_count)
print
print "Are you sure you want to continue? [Y/n]"
user_input = raw_input("> ")
return (user_input.upper() in ("", "Y", "YES"))
else:
print
print "Nothing to download."
return False
def confirm_download_schedule(schedule):
"""Reports the total number of bytes and total number of files
to download. Also lists the inaccessible files (based on HEAD
response). Then asks user to confirm downloading.
"""
def content_length(tpl):
return tpl[2][1]
def status_code(tpl):
return tpl[2][0]
def href(tpl):
return tpl[0]
def is_OK(tpl):
return status_code(tpl) == 200
def not_OK(tpl):
return status_code(tpl) != 200
increment = lambda x, _: x + 1
file_count = schedule >> stream.filter(is_OK) >> stream.reduce(increment, 0)
bytes_to_download = schedule >> stream.filter(is_OK) >> stream.map(content_length) >> sum
inaccessible_files = schedule >> stream.filter(not_OK) >> list
if len(inaccessible_files) > 0:
print
print "Some files are inaccessible:"
for (idx, sched) in enumerate(inaccessible_files):
print "%d: %d %s" % (idx, status_code(sched), href(sched))
if bytes_to_download > 0:
print
print "Need to download %s in %d files." % (pretty_bytes(bytes_to_download), file_count)
print
print "Are you sure you want to continue? [Y/n]"
user_input = raw_input("> ")
return user_input.upper() in ("", "Y", "YES")
else:
print
print "Nothing to download."
return False
def test_ForkedFeeder():
result = stream.ForkedFeeder(blocking_producer) >> stream.map(f) >> stream.reduce(operator.add)
pprint(result)
assert result == expected
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
import stream
## Test scenario based on ../example/feeder.py
def blocking_producer():
for n in range(25):
time.sleep(0.01)
yield 42
f = lambda x: x ** 2
expected = blocking_producer() >> stream.map(f) >> stream.reduce(operator.add)
## Test cases
def test_ThreadedFeeder():
result = stream.ThreadedFeeder(blocking_producer) >> stream.map(f) >> stream.reduce(operator.add)
pprint(result)
assert result == expected
def test_ForkedFeeder():
result = stream.ForkedFeeder(blocking_producer) >> stream.map(f) >> stream.reduce(operator.add)
pprint(result)
assert result == expected
user 0m7.046s
sys 0m0.020s
$ time python ./feeder.py -s # sequential
real 0m13.072s
user 0m7.596s
sys 0m0.067s
"""
def blocking_producer():
for n in range(25):
time.sleep(0.01)
yield 42
if __name__ == '__main__':
f = lambda x: x**x**3
import sys
try:
if sys.argv[1] == '-s':
## use a single thread
blocking_producer() >> map(f) >> reduce(operator.add)
elif sys.argv[1] == '-t':
## use a feeder in a separate thread
ThreadedFeeder(blocking_producer) >> map(f) >> reduce(operator.add)
except IndexError:
## use a feeder in a child process
ForkedFeeder(blocking_producer) >> map(f) >> reduce(operator.add)
def test_ForkedFeeder():
result = ForkedFeeder(blocking_producer) >> map(f) >> reduce(operator.add)
pprint(result)
assert result == expected
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
from stream import ThreadedFeeder, ForkedFeeder, map, reduce
## Test scenario based on ../example/feeder.py
def blocking_producer():
for n in range(25):
time.sleep(0.01)
yield 42
f = lambda x: x**2
expected = blocking_producer() >> map(f) >> reduce(operator.add)
## Test cases
def test_ThreadedFeeder():
result = ThreadedFeeder(blocking_producer) >> map(f) >> reduce(
operator.add)
pprint(result)
assert result == expected
def test_ForkedFeeder():
result = ForkedFeeder(blocking_producer) >> map(f) >> reduce(operator.add)
pprint(result)
assert result == expected
def test_ThreadedFeeder(): result = ThreadedFeeder(blocking_producer) >> map(f) >> reduce(operator.add) pprint(result) assert result == expected
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) from stream import ThreadedFeeder, ForkedFeeder, map, reduce ## Test scenario based on ../example/feeder.py def blocking_producer(): for n in range(25): time.sleep(0.01) yield 42 f = lambda x: x**2 expected = blocking_producer() >> map(f) >> reduce(operator.add) ## Test cases def test_ThreadedFeeder(): result = ThreadedFeeder(blocking_producer) >> map(f) >> reduce(operator.add) pprint(result) assert result == expected def test_ForkedFeeder(): result = ForkedFeeder(blocking_producer) >> map(f) >> reduce(operator.add) pprint(result) assert result == expected
def test_ThreadedFeeder():
result = stream.ThreadedFeeder(blocking_producer) >> stream.map(
f) >> stream.reduce(operator.add)
pprint(result)
assert result == expected
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
import stream
## Test scenario based on ../example/feeder.py
def blocking_producer():
for n in range(25):
time.sleep(0.01)
yield 42
f = lambda x: x**2
expected = blocking_producer() >> stream.map(f) >> stream.reduce(operator.add)
## Test cases
def test_ThreadedFeeder():
result = stream.ThreadedFeeder(blocking_producer) >> stream.map(
f) >> stream.reduce(operator.add)
pprint(result)
assert result == expected
def test_ForkedFeeder():
result = stream.ForkedFeeder(blocking_producer) >> stream.map(
f) >> stream.reduce(operator.add)
pprint(result)
real 0m7.231s user 0m7.046s sys 0m0.020s $ time python ./feeder.py -s # sequential real 0m13.072s user 0m7.596s sys 0m0.067s """ def blocking_producer(): for n in range(25): time.sleep(0.01) yield 42 if __name__ == '__main__': f = lambda x: x**x**3 import sys try: if sys.argv[1] == '-s': ## use a single thread blocking_producer() >> map(f) >> reduce(operator.add) elif sys.argv[1] == '-t': ## use a feeder in a separate thread ThreadedFeeder(blocking_producer) >> map(f) >> reduce(operator.add) except IndexError: ## use a feeder in a child process ForkedFeeder(blocking_producer) >> map(f) >> reduce(operator.add)