def main():
"""Funcion principal, donde interactuamos con las
demas clases pasandoles los ficheros necesarios e
instanciando las clases del MapReduce"""
num_cores = 4
files = getArgs()
file_manager = FileManager(files)
lines_files = file_manager.split_in_lines()
num_lines = len(lines_files)
partialPart = num_lines / num_cores
difference = num_lines - (partialPart * num_cores)
mapper = Mapper("")
for i in range(partialPart, (num_lines - partialPart) + 1, partialPart):
t = threading.Thread(mapper.mapping(lines_files[i - partialPart:i]))
t.start()
t = threading.Thread(
mapper.mapping(lines_files[num_lines -
(partialPart + difference):num_lines]))
t.start()
shuffleDict = mapper.shuffle(mapper.wordsMap)
reducer = Reducer()
result = reducer.reduce(shuffleDict)
dirToTxt(result)
def test_reduce(self):
# TODO: Remove BOM for csv file to display MSISDN
data_frame = pd.DataFrame(self.test_dict)
test_reducer = Reducer.Reducer(data_frame)
generated_reduced_data_frame = test_reducer.reduce()
reduced_dict = [{
'Date': datetime.datetime(2016, 3, 1),
'Network': 'Network 1',
'Product': 'Loan Product 1',
'Aggregate': 1000,
'Count': 1
}, {
'Date': datetime.datetime(2016, 3, 1),
'Network': 'Network 2',
'Product': 'Loan Product 1',
'Aggregate': 1122,
'Count': 1
}, {
'Date': datetime.datetime(2016, 3, 1),
'Network': 'Network 3',
'Product': 'Loan Product 2',
'Aggregate': 2084,
'Count': 1
}]
reduced_data_frame = pd.DataFrame(reduced_dict)
grouped_by_month = reduced_data_frame.Date.dt.to_period("M")
group_by_tuple = reduced_data_frame.groupby(
[grouped_by_month, 'Network', 'Product'])
expected_reduced_data_frame = group_by_tuple.sum()
assert generated_reduced_data_frame.equals(expected_reduced_data_frame)
def calculate_aggregate_loans(self):
# Mapper object
aggregator_map = Mapper.Mapper(self.input_path)
data = aggregator_map.create_data_frame()
# Reducer object
reducer = Reducer.Reducer(data)
reducer.reduce()
reducer.print_to_csv(self.output_path)
print("Each row requires " + str(columns) + " arguments. Try again!")
return matrix
if __name__ == '__main__':
matrix = takeInput()
print()
print('Given matrix:')
for row in matrix:
print(row)
print()
rd = ReducerModule.Reducer(matrix)
print('Echelon form:')
matrix = rd.EchelonForm()
for row in matrix:
print(["{0:0.3f}".format(element) for element in row], end='')
print('\t', end='')
print([str(Fraction(element).limit_denominator()) for element in row])
print()
print('Reduced Echelon form:')
matrix = rd.ReducedEchelonForm()
for row in matrix:
print(["{0:0.3f}".format(element) for element in row], end='')
print('\t', end='')
print([str(Fraction(element).limit_denominator()) for element in row])
if not Utils.verify(output, info["verification"]):
exit("Reduced testcase doesn't trigger bug on buggy js parser.")
output = Utils.run(info["js"], info["reduced_test"])
if output == -1:
exit("Reduced testcase gives an error when run.")
if Utils.verify(output, info["verification"]):
exit("Reduced testcase triggers bug on normal js parser")
# Reduce
f = open(info["reduced_test"], 'r')
lines = len(f.readlines())
f.close()
script = Script(info)
reducer = Reducer(script)
#reducer.start()
import random
while 1:
length = random.randint(0, 5) + 1
line = random.randint(0, len(reducer.script.lines) - 5)
print line, line + length
lines = reducer.script.lines
reducer.line1(line, line + length)
"""
for i in range(100, 0, -1):
#for j in range(lines/i):
for j in range(lines/i, 0, -1):
print j*i, (j+1)*i
# -*- coding: utf-8 -*-
"""
Created on Sat Jan 23 19:04:52 2021
@author: Aravindh
"""
import pandas
import Reducer as red
MyList = []
def Mapper():
ds = pandas.read_csv("Project.csv")
for line in ds.index:
price = (ds['Price'][line]).replace(",", "")
country = ds['Country'][line]
product = ds['Product'][line]
MyList.append(list((country, product, int(price))))
Mapper()
red.Reducer(MyList)
red.printDictionary()
