print 'Fetching', site
u = urllib.urlopen(site)
'''
Program
|
/ \
Thread Processes
| |
/ \ \
page len len (BEST due to IO operation is multiproceesing now)
'''
# Returning entire website
# return site, u.read()
# Optimized?? Returning only len... NO because This is multithreading, it's already shared
return site, len(u.read())
def mymap(func, iterable):
return [func(x) for x in iterable]
# Everything shared should be done before Pool() to use COW (copy on write)
pool = Pool(20)
pprint(zip(sites, imap(len, imap(get_site, sites))))
for site, size in pool.imap_unorderd(get_site, sites): # imap_unordered generates a generator
print size, site
