def Map_WC(L):
results = []
for w in L:
# True if w contains non-alphanumeric characters
if not w.isalnum():
w = MR_u.sanitize(w)
# True if w is a title-cased token
if w.istitle():
results.append((w, 1))
return results
# python MapReduce_main.py test.txt
import sys
import MapReduce_util as MR_u
import MapReduce_func as MR_f
from multiprocessing import Pool
if len(sys.argv) != 2:
print "Program requires path to file for reading!"
sys.exit(1)
num_proc = 1
# Load file, stuff it into a string
text = MR_u.load(sys.argv[1])
# Build a pool of 8 processes
pool = Pool(processes=num_proc)
# Fragment the string data into 8 chunks
partitioned_text = list(MR_u.chunks(text, len(text) / num_proc))
# print len(text) / num_proc,len(partitioned_text[0])
# Generate count tuples for title-cased tokens
single_count_tuples = pool.map(MR_f.Map_WC, partitioned_text)
# single_count_tuples = pool.map(MR_f.Map_WC, partitioned_text)
"""
map_func = MR_f.Mapper('WordCount')
single_count_tuples = map_func.Map(partitioned_text[0])
token_to_tuples = MR_f.Partition([single_count_tuples])
"""
# print len(single_count_tuples)#
# Organize the count tuples; lists of tuples by token key
