def master(x, ibm_cos):
obj = COSBackend(config=ibm_cos)
def order(e):
return e['LastModified']
write_permission_list = []
m = []
finish = 0
obj.put_object('practise2', 'result.json', json.dumps(m))
l = obj.list_objects('practise2', 'p_write')
while (not finish):
l.sort(key=order)
current_id = l.pop(0)
current_id = current_id['Key']
file_to_write = current_id[2:]
date_json = obj.list_objects('practise2',
'result.json')[0]['LastModified']
obj.put_object('practise2', file_to_write, "")
obj.delete_object('practise2', "p_" + file_to_write)
write_permission_list.append(int(file_to_write[7:-1]))
next = 0
while (not next):
time.sleep(X / 4)
if (not obj.list_objects('practise2',
'result.json')[0]['LastModified']
== date_json):
next = 1
obj.delete_object('practise2', file_to_write)
time.sleep(X)
l = obj.list_objects('practise2', 'p_write')
if (not l):
finish = 1
return write_permission_list
def reduce_word_count(args):
file_name = args.get('file_name')
num_partitions = args.get('num_partitions')
cos = COSBackend(args.get('cos_params'))
bucket_name = args.get('bucket_name')
result_dict = {}
for i in range(num_partitions):
file = "wc_" + file_name + str(i)
file_dict = json.loads(cos.get_object(bucket_name, file))
cos.delete_object(bucket_name, file)
result_dict = {
key: result_dict.get(key, 0) + file_dict.get(key, 0)
for key in set(result_dict) | set(file_dict)
}
cos.put_object(bucket_name, "final_" + file_name, json.dumps(result_dict))
return {'finish': "OK"}
def reduce_count_words(args):
file_name = args.get('file_name')
num_partitions = args.get('num_partitions')
cos = COSBackend(args.get('cos_params'))
bucket_name = args.get('bucket_name')
total_words = 0
for i in range(num_partitions):
file = "cw_" + file_name + str(i)
total_words += int(cos.get_object(bucket_name, file))
cos.delete_object(bucket_name, file)
cos.put_object(bucket_name, "final_" + file_name, str(total_words))
#for i in range(num_partitions):
# file_to_delete = "cw_"+file_name+str(i)
# cos.delete_object(bucket_name, file_to_delete)
return {'finish': "OK"}
def clean():
cos = COSBackend()
print('Cleaning...', end='')
if WORKERS != 1:
cos.delete_object(BUCKET, '/secuencial/B')
for i in range(0, WORKERS):
print('.', end='')
cos.delete_object(BUCKET, '/paralelo/f' + str(i))
else:
cos.delete_object(BUCKET, '/secuencial/A')
cos.delete_object(BUCKET, '/secuencial/B')
print('.', end='\n')
if __name__ == '__main__':
cos=COSBackend()
ibcmf= pywren.ibm_cf_executor()
start_time = time.time()
ibcmf.wait(ibcmf.call_async(generatex,[m,n,l,a]))
ibcmf.clean()
iterdata=[]
for i in range(nWorkersA):
for j in range(nWorkersB):
iterdata.append(f'{i}w{j}')
#start_time = time.time()
ibcmf.wait(ibcmf.map_reduce(matrix_multiplication,iterdata, multiplication_reduce, reducer_wait_local=True))
elapsed_time = time.time() - start_time
for i in iterdata:
cos.delete_object('practica-sd-mp',i)
matrixA=pickle.loads(cos.get_object('practica-sd-mp','matrixA.txt'))
matrixB=pickle.loads(cos.get_object('practica-sd-mp','matrixB.txt'))
matrixC=pickle.loads(cos.get_object('practica-sd-mp','matrixC.txt'))
print(f'Matriu A ({m} x {n}):')
for filaA in matrixA:
print(filaA)
print(f'Matriu B ({n} x {l}):')
for filaB in matrixB:
print(filaB)
print(f'Matriu C ({m} x {l}):' )
for filaC in matrixC:
print(filaC)
print(f'Valor de m: {m}\nValor de n: {n}\nValor de l: {l}\nValor de a: {a}')