def main(): """Funcion principal, donde interactuamos con las demas clases pasandoles los ficheros necesarios e instanciando las clases del MapReduce""" num_cores = 4 files = getArgs() file_manager = FileManager(files) lines_files = file_manager.split_in_lines() num_lines = len(lines_files) partialPart = num_lines / num_cores difference = num_lines - (partialPart * num_cores) mapper = Mapper("") for i in range(partialPart, (num_lines - partialPart) + 1, partialPart): t = threading.Thread(mapper.mapping(lines_files[i - partialPart:i])) t.start() t = threading.Thread( mapper.mapping(lines_files[num_lines - (partialPart + difference):num_lines])) t.start() shuffleDict = mapper.shuffle(mapper.wordsMap) reducer = Reducer() result = reducer.reduce(shuffleDict) dirToTxt(result)
def test_reduce(self): # TODO: Remove BOM for csv file to display MSISDN data_frame = pd.DataFrame(self.test_dict) test_reducer = Reducer.Reducer(data_frame) generated_reduced_data_frame = test_reducer.reduce() reduced_dict = [{ 'Date': datetime.datetime(2016, 3, 1), 'Network': 'Network 1', 'Product': 'Loan Product 1', 'Aggregate': 1000, 'Count': 1 }, { 'Date': datetime.datetime(2016, 3, 1), 'Network': 'Network 2', 'Product': 'Loan Product 1', 'Aggregate': 1122, 'Count': 1 }, { 'Date': datetime.datetime(2016, 3, 1), 'Network': 'Network 3', 'Product': 'Loan Product 2', 'Aggregate': 2084, 'Count': 1 }] reduced_data_frame = pd.DataFrame(reduced_dict) grouped_by_month = reduced_data_frame.Date.dt.to_period("M") group_by_tuple = reduced_data_frame.groupby( [grouped_by_month, 'Network', 'Product']) expected_reduced_data_frame = group_by_tuple.sum() assert generated_reduced_data_frame.equals(expected_reduced_data_frame)
def calculate_aggregate_loans(self): # Mapper object aggregator_map = Mapper.Mapper(self.input_path) data = aggregator_map.create_data_frame() # Reducer object reducer = Reducer.Reducer(data) reducer.reduce() reducer.print_to_csv(self.output_path)
print("Each row requires " + str(columns) + " arguments. Try again!") return matrix if __name__ == '__main__': matrix = takeInput() print() print('Given matrix:') for row in matrix: print(row) print() rd = ReducerModule.Reducer(matrix) print('Echelon form:') matrix = rd.EchelonForm() for row in matrix: print(["{0:0.3f}".format(element) for element in row], end='') print('\t', end='') print([str(Fraction(element).limit_denominator()) for element in row]) print() print('Reduced Echelon form:') matrix = rd.ReducedEchelonForm() for row in matrix: print(["{0:0.3f}".format(element) for element in row], end='') print('\t', end='') print([str(Fraction(element).limit_denominator()) for element in row])
if not Utils.verify(output, info["verification"]): exit("Reduced testcase doesn't trigger bug on buggy js parser.") output = Utils.run(info["js"], info["reduced_test"]) if output == -1: exit("Reduced testcase gives an error when run.") if Utils.verify(output, info["verification"]): exit("Reduced testcase triggers bug on normal js parser") # Reduce f = open(info["reduced_test"], 'r') lines = len(f.readlines()) f.close() script = Script(info) reducer = Reducer(script) #reducer.start() import random while 1: length = random.randint(0, 5) + 1 line = random.randint(0, len(reducer.script.lines) - 5) print line, line + length lines = reducer.script.lines reducer.line1(line, line + length) """ for i in range(100, 0, -1): #for j in range(lines/i): for j in range(lines/i, 0, -1): print j*i, (j+1)*i
# -*- coding: utf-8 -*- """ Created on Sat Jan 23 19:04:52 2021 @author: Aravindh """ import pandas import Reducer as red MyList = [] def Mapper(): ds = pandas.read_csv("Project.csv") for line in ds.index: price = (ds['Price'][line]).replace(",", "") country = ds['Country'][line] product = ds['Product'][line] MyList.append(list((country, product, int(price)))) Mapper() red.Reducer(MyList) red.printDictionary()