def run_reducer(self, index):
"""Runs the implemented reducer
"""
key_values_map = {}
for mapper_index in range(self.n_mappers):
temp_map_file = open(settings.get_temp_map_file(mapper_index, index), "r")
mapper_results = json.load(temp_map_file)
for (key, value) in mapper_results:
if(not(key in key_values_map)):
key_values_map[key]= []
try:
key_values_map[key].append(value)
except Exception, e:
print "Exception while inserting key: "+ str(e)
temp_map_file.close()
if(self.clean):
os.unlink(settings.get_temp_map_file(mapper_index, index))
def run_reducer(self, index):
"""Runs the implemented reducer
:param index: the index of the thread to run on
"""
key_values_map = {}
for mapper_index in range(self.n_mappers):
temp_map_file = open(
settings.get_temp_map_file(mapper_index, index), "r")
mapper_results = json.load(temp_map_file)
for (key, value) in mapper_results:
if not (key in key_values_map):
key_values_map[key] = []
try:
key_values_map[key].append(value)
except Exception, e:
print "Exception while inserting key: " + str(e)
temp_map_file.close()
if self.clean:
os.unlink(settings.get_temp_map_file(mapper_index, index))
def run_mapper(self, index):
"""Runs the implemented mapper
"""
input_split_file = open(settings.get_input_split_file(index), "r")
key = input_split_file.readline()
value = input_split_file.read()
input_split_file.close()
if(self.clean):
os.unlink(settings.get_input_split_file(index))
mapper_result = self.mapper(key, value)
for reducer_index in range(self.n_reducers):
temp_map_file = open(settings.get_temp_map_file(index, reducer_index), "w+")
json.dump([(key, value) for (key, value) in mapper_result
if self.check_position(key, reducer_index)]
, temp_map_file)
temp_map_file.close()
def run_mapper(self, index):
"""Runs the implemented mapper
:param index: the index of the thread to run on
"""
input_split_file = open(settings.get_input_split_file(index), "r")
key = input_split_file.readline()
value = input_split_file.read()
input_split_file.close()
if (self.clean):
os.unlink(settings.get_input_split_file(index))
mapper_result = self.mapper(key, value)
for reducer_index in range(self.n_reducers):
temp_map_file = open(
settings.get_temp_map_file(index, reducer_index), "w+")
json.dump([(key, value) for (key, value) in mapper_result
if self.check_position(key, reducer_index)],
temp_map_file)
temp_map_file.close()