for k in data_2.keys():
if k in data_1:
new_list = data_1[k] + data_2[k]
data_1[k] = new_list
else:
data_1[k] = data_2[k]
return data_1
job_name = sys.argv[1]
chunk_size = int(sys.argv[2])
no_of_reducers = int(sys.argv[3])
input_file = sys.argv[4]
output_file = sys.argv[5]
reducer_list = list(range(0, no_of_reducers))
splits = splitter.split_file(input_file, chunk_size)
engine = None
def silentremove(filename):
try:
os.remove(filename)
except OSError as e: # this would be "except OSError, e:" before Python 2.6
# if e.errno != errno.ENOENT: # errno.ENOENT = no such file or directory
pass # re-raise exception if a different error occured
# Map Phase
for start_index in splits:
if job_name == jobs.Jobs.SORTING_JOB or job_name == jobs.Jobs.WORD_COUNT_JOB:
values = splitter.read_chunk_by_word(input_file, start_index, chunk_size)
for k in data_2.keys():
if k in data_1:
new_list = data_1[k] + data_2[k]
data_1[k] = new_list
else:
data_1[k] = data_2[k]
return data_1
job_name = sys.argv[1]
chunk_size = int(sys.argv[2])
no_of_reducers = int(sys.argv[3])
input_file = sys.argv[4]
output_file = sys.argv[5]
reducer_list = list(range(0, no_of_reducers))
splits = splitter.split_file(input_file, chunk_size)
engine = None
def silentremove(filename):
try:
os.remove(filename)
except OSError as e: # this would be "except OSError, e:" before Python 2.6
# if e.errno != errno.ENOENT: # errno.ENOENT = no such file or directory
pass # re-raise exception if a different error occured
# Map Phase
for start_index in splits:
if job_name == jobs.Jobs.SORTING_JOB or job_name == jobs.Jobs.WORD_COUNT_JOB:
values = splitter.read_chunk_by_word(input_file, start_index,
def split_input(self, input_file, chunk_size):
return splitter.split_file(input_file, chunk_size)