def test_per_worker_score(): file1 = STORAGE_PATH + "file1.txt" file2 = STORAGE_PATH + "file2.txt" createNewFile(file1) print("Created new file 1") createNewFile(file2) print("Created new file 2") range1 = 8 for i in range(range1): increment1(i, file1) increment2(i, file2) file1 = compss_wait_on(file1) file2 = compss_wait_on(file2) for i in range(range1): increment1(i, file1) increment2(i, file2) with compss_open(file1, 'r') as fis: final_value1 = fis.read() with compss_open(file2, 'r') as fis: final_value2 = fis.read() #compss_wait_on(result1) #compss_wait_on(result2) print("Result 1 = " + str(final_value1)) print("Result 2 = " + str(final_value2))
def check_decorators(): binary_result = "binary_result.out" mpi_result = "mpi_result.out" ompss_result = "ompss_result.out" check_binary(binary_result) check_mpi(mpi_result) check_ompss(ompss_result) compss_wait_on_file(binary_result) compss_wait_on_file(mpi_result) compss_wait_on_file(ompss_result) binary_result_fd = compss_open(binary_result) mpi_result_fd = compss_open(mpi_result) ompss_result_fd = compss_open(ompss_result) binary_content = binary_result_fd.readlines() mpi_content = mpi_result_fd.readlines() ompss_content = ompss_result_fd.readlines() binary_result_fd.close() mpi_result_fd.close() ompss_result_fd.close() os.remove(binary_result) os.remove(mpi_result) os.remove(ompss_result) assert len(binary_content) == 1 assert len(mpi_content) == 1 assert len(ompss_content) == 1 print(binary_content) print(mpi_content) print(ompss_content)
def testMultiReturnFile(self): """ Test multireturn FILE """ fretout1 = "retFile1" fretout2 = "retFile2" content1 = "RETURN FILE CONTENTS A" content2 = "RETURN FILE CONTENTS B" retResA, retResB = self.multireturnFile(fretout1, fretout2, content1, content2) retResA = compss_wait_on(retResA) retResB = compss_wait_on(retResB) with compss_open(retResA, 'r') as f: content1_r = f.read() with compss_open(retResB, 'r') as f: content2_r = f.read() self.assertEqual(res, fret, "strings are not equal: {}, {}".format(content2_r, content2)) self.assertEqual(content_r, content, "strings are not equal: {}, {}".format(content1_r, content1))
def load_file(self, file_path, chunk_size=1024, worker_read=False): """ Read file in chunks and save it onto partitions. :param file_path: a path to a file to be loaded :param chunk_size: size of chunks in bytes :param worker_read: :return: >>> with open("test.file", "w") as testFile: ... _ = testFile.write("Hello world!") >>> DDS().load_file("test.file", 6).collect() ['Hello ', 'world!'] """ if worker_read: fp = open(file_path) fp.seek(0, 2) total = fp.tell() parsed = 0 while parsed < total: self.partitions.append( load_partition_from_file(file_path, parsed, chunk_size)) parsed += chunk_size else: f = compss_open(file_path, 'r') chunk = f.read(chunk_size) while chunk: self.partitions.append(task_load(chunk)) chunk = f.read(chunk_size) return self
def test_simple(self): from pycompss.api.api import compss_open initial_value = '1' file_name = "counter" # Write value fos = open(file_name, 'w') fos.write(initial_value) fos.close() print("Initial counter value is " + initial_value) # Execute increment increment(file_name) # Write new value fis = compss_open(file_name, 'r+') final_value = fis.read() fis.close() print("Final counter value is " + final_value) expected_final_value = '2' if final_value != expected_final_value: print("Simple increment test result is incorrect. " "Expected: %s, got: %s" % (expected_final_value, final_value)) self.assertEqual(final_value, expected_final_value)
def test_mp_file_access(): print "test_file_mp_access" print "test_function_files" fin = 'infile' finout = 'inoutfile' fout = 'outfile' fin_d = open(fin, 'w') finout_d = open(finout, 'w') fin_d.write('IN FILE CONTENT') finout_d.write('INOUT FILE INITIAL CONTENT') fin_d.close() finout_d.close() function_files(fin, finout, fout) fout_d = compss_open(fout, 'r+') if fout_d.read() == 'OUT FILE CONTENT': print "- File access from MP: OK" else: print "- File access from MP: ERROR" fout_d.write("\n===> OUT FILE ADDED CONTENT") fout_d.close() function_files(fout, finout, fin)
def testFileIN_OUTDuplic(self): """ Test FILE_OUT """ from pycompss.api.api import compss_wait_on from pycompss.api.api import compss_open if not os.path.exists("id1"): os.mkdir("id1") if not os.path.exists("id2"): os.mkdir("id2") fin = "id1/outfile" fout = "id2/outfile" content_1 = "IN FILE CONTENT" content_2 = "OUT FILE CONTENT" self.fileOut(fin, content_1) res1, res2 = self.fileInOutDuplicated(fin, fout, "outfile", content_2) res1 = compss_wait_on(res1) with compss_open(fout, 'r') as fout_r: content_r = fout_r.read() # The final file is only stored after the execution. # During the execution, you have to use the compss_open, which will # provide the real file where the output file is. # fileInFolder = os.path.exists(fout) # self.assertTrue(fileInFolder, "FILE_OUT is not in the final location") self.assertEqual( res1, content_1, "strings are not equal: {}, {}".format(res1, content_1)) self.assertEqual( content_r, content_2, "strings are not equal: {}, {}".format(content_r, content_2))
def test_default_file_out(self): initial_file = "my_out_file.txt" i_will_fail_file_out(initial_file) with compss_open(initial_file) as f: content = f.read() assert content == "EMPTY FILE OUT", "ERROR: Wrong file inout (%s != EMPTY FILE OUT)" % content compss_delete_file(initial_file)
def test_mp_file_access(): print("test_file_mp_access") print("test_function_files") fin = 'infile' finout = 'inoutfile' fout = 'outfile' fin_d = open(fin, 'w') finout_d = open(finout, 'w') fin_d.write('IN FILE CONTENT') finout_d.write('INOUT FILE INITIAL CONTENT') fin_d.close() finout_d.close() function_files(fin, finout, fout) fout_d = compss_open(fout, 'r+') if fout_d.read() == 'OUT FILE CONTENT': print("- File access from MP: OK") else: print("- File access from MP: ERROR") fout_d.write("\n===> OUT FILE ADDED CONTENT") fout_d.close() function_files(fout, finout, fin)
def testFileIN_INOUTDuplic(self): """ Test FILE_INOUT """ from pycompss.api.api import compss_wait_on from pycompss.api.api import compss_open if not os.path.exists("id1"): os.mkdir("id1") if not os.path.exists("id2"): os.mkdir("id2") fin = "id1/inoutfile" finout = "id2/inoutfile" content1 = "INOUT FILE CONTENT 1" content2 = "INOUT FILE CONTENT 2" self.fileOut(fin, content1) self.fileOut(finout, content2) res1, res2 = self.fileInInoutDpulicated(fin, finout, "inoutfile") res1 = compss_wait_on(res1) with compss_open(finout, 'r') as finout_r: content_r = finout_r.read() content2 += "\n===> INOUT FILE ADDED CONTENT" self.assertEqual( res1, content1, "strings 1 are not equal: {}, {}".format(res1, content1)) self.assertEqual( content_r, content2, "strings 2 are not equal: {}, {}".format(content_r, content2))
def testDeleteFile(self): # Check and get parameters initial_value = '1' counter_name = 'counter_INOUT' # check that this file does not exist after the execution counter_name_IN = 'counter_IN' # check that this file does not exist after the execution counter_name_OUT = 'counter_OUT' # check that this file does not exist after the execution for i in range(3): # Write value if i <= 1: fos = open(counter_name, 'w') fos.write(initial_value) fos.close() fos2 = open(counter_name_IN, 'w') fos2.write(initial_value) fos2.close() print(('Initial counter value is %s' % initial_value)) # Execute increment increment(counter_name) increment2(counter_name_IN, counter_name_OUT) # Read new value print('After sending task') if i == 0: compss_delete_file(counter_name) compss_delete_file(counter_name_IN) compss_delete_file(counter_name_OUT) fis = compss_open(counter_name, 'r+') final_value = fis.read() fis.close() print(('Final counter value is %s' % final_value)) self.assertEqual(final_value, '3') compss_delete_file(counter_name)
def main_complex_example(): # Imports from pycompss.api.api import compss_open # LAUNCH COMPLEX EXAMPLE if __debug__: print("Launching MPI complex example") # Constants n = 4 m = 8 block_size = 16 # Generate data data = [[None for _ in range(m)] for _ in range(n)] for i in range(n): for j in range(m): data[i][j] = "file_" + str(i) + "_" + str(j) init_data_block_in_file(data[i][j], block_size) # Process data compact_data = [None for _ in range(n)] final_data = "final_data" for i in range(n): compact_data[i] = "compact_" + str(i) mpi_compact(compact_data[i], *data[i]) mpi_compact(final_data, *compact_data) # Synchronise with compss_open(final_data, "r") as f: print(f.read()) # DONE if __debug__: print("DONE Launching MPI complex example")
def testFileManagementINOUT(self): inoutfile = "src/inoutfile" mySedINOUT('-i', 's/Hi/HELLO/g', inoutfile) with compss_open(inoutfile, "r") as finout_r: content_r = finout_r.read() # Check if there are no Hi words, and instead there is HELLO if 'Hi' in content_r: self.fail("INOUT File failed.")
def test_default_file_inout(self): initial_file = "my_inout_file.txt" with open(initial_file, 'w') as f: f.write("INITIAL FILE INOUT") i_will_fail_file_inout(initial_file) with compss_open(initial_file) as f: content = f.read() assert content == "EMPTY FILE INOUT", "ERROR: Wrong file inout (%s != EMPTY FILE INOUT)" % content compss_delete_file(initial_file)
def printCounterValue(filePath, pos): from pycompss.api.api import compss_open # Read value counter 1 fis = compss_open(filePath, 'r+') counter = fis.read() fis.close() # Print values print "- Counter" + str(pos) + " value is " + counter
def test_file_dependencies_complex(self): """ Creates a file on a task, verifies its content in a second one, a third task updates its value, and, then, the master checks the value. After that, the master updates the value, verifies its content locally and on a task. Later, it updates the value on a task, checks the value on another task, and updates twice the value on using two tasks. Finally, the value returns to the master so it checks it. """ print("Testing file dependencies - Complex Version") file_name = "dependencies_file_2" create_file_with_content(INITIAL_CONTENT, file_name) check_file_with_content(INITIAL_CONTENT, file_name) check_and_update_file_with_content(INITIAL_CONTENT, UPDATED_CONTENT_1, file_name) with compss_open(file_name, "r") as f_channel: line = f_channel.readline() verify_line(line, UPDATED_CONTENT_1) # Update File Content on Main with compss_open(file_name, "w") as f_channel: f_channel.write(UPDATED_CONTENT_2) # Verify File update on Main with compss_open(file_name, "r") as f_channel: line = f_channel.readline() verify_line(line, UPDATED_CONTENT_2) check_file_with_content(UPDATED_CONTENT_2, file_name) check_and_update_file_with_content(UPDATED_CONTENT_2, UPDATED_CONTENT_3, file_name) check_file_with_content(UPDATED_CONTENT_3, file_name) check_and_update_file_with_content(UPDATED_CONTENT_3, UPDATED_CONTENT_4, file_name) check_and_update_file_with_content(UPDATED_CONTENT_4, UPDATED_CONTENT_5, file_name) with compss_open(file_name, "r") as f_channel: line = f_channel.readline() verify_line(line, UPDATED_CONTENT_5) compss_barrier() compss_delete_file(file_name) print("\t OK")
def testCheckFileNames(self): f = "src/infile" fp = "src/infile" name = "infile" fout = "checkFileNamesResult.txt" exit_value = checkFileNames(f, fp, name, fout) exit_value = compss_wait_on(exit_value) with compss_open(fout) as result: data = result.read() print("CheckFileNamesResult: " + str(data)) self.assertEqual(exit_value, 0, "At least one file name is NOT as expected: {}, {}, {}".format(f, fp, name))
def testReturnFile(self): """ Test return FILE """ from pycompss.api.api import compss_wait_on fret = "fret" content = "RETURN FILE CONTENT" res = self.returnFile(fret, content) res = compss_wait_on(res) with compss_open(res, 'r') as f: content_r = f.read() self.assertEqual(res, fret, "strings are not equal: {}, {}".format(res, content)) self.assertEqual(content_r, content, "strings are not equal: {}, {}".format(content_r, content))
def test_binary_INOUT(self): message = 'Hi, this is a test.' f = "myfile.txt" create_input(message, f) mySedINOUT('-i', 's/Hi/HELLO/g', f) with compss_open(f, "r") as f_result: content_r = f_result.read() # Check if there are no Hi words, and instead there is HELLO if 'Hi' in content_r: self.fail("INOUT File failed.") if not 'HELLO' in content_r: self.fail("INOUT File failed.")
def printCounterValue(filePath, pos): from pycompss.api.api import compss_open # Read value counter 1 fis = compss_open(filePath, 'r') counter = fis.read() fis.close() # Print values print("- Counter" + str(pos) + " value is " + counter) expected = INITIAL_VALUE + 1 if int(counter) != expected: print(" -Incorrect counter value " + counter + " expected " + str(expected)) exit(-1)
def test_function_files(self): """ test function files """ from pycompss.api.api import compss_open # todo hacer tests separados fin, finout, fout = 'infile', 'inoutfile', 'outfile' fin_d = open(fin, 'w') finout_d = open(finout, 'w') fin_d.write('IN FILE CONTENT') finout_d.write('INOUT FILE INITIAL CONTENT') fin_d.close() finout_d.close() self.function_files(fin, finout, fout) # add check content fin_d = open(fin, 'r') self.assertEqual(fin_d.read(), 'IN FILE CONTENT') finout_d = compss_open(finout, 'r') self.assertEqual( finout_d.read(), 'INOUT FILE INITIAL CONTENT\n===> INOUT FILE ADDED CONTENT') fout_d = compss_open(fout, 'r') self.assertEqual(fout_d.read(), "OUT FILE CONTENT")
def test_task_to_main(self): """ Creates a file on a task and reads it on the master. """ print("Creating file on task and using it on main") file_name = "task_to_main_file" create_file_with_content(INITIAL_CONTENT, file_name) with compss_open(file_name, "r") as f_channel: line = f_channel.readline() verify_line(line, INITIAL_CONTENT) line = f_channel.readline() verify_line(line, None) compss_barrier() compss_delete_file(file_name) print("\t OK")
def test_binary_container(self): # Imports from pycompss.api.api import compss_barrier from pycompss.api.api import compss_wait_on from pycompss.api.api import compss_open # Test empty binary execution task_binary_empty() compss_barrier() # Test exit value ev = task_binary_ev() ev = compss_wait_on(ev) self.assertEquals(ev, 0) # Test working dir # WARN: Check WD in result script task_binary_wd() compss_barrier() # Test stdout and stderr stdout = "binary_output.txt" stderr = "binary_error.txt" task_binary_std(stdout, stderr) print("STDOUT:") with compss_open(stdout, 'r') as f: content = f.read() print(content) self.assertTrue(len(content) != 0) print("STDERR:") with compss_open(stderr, 'r') as f: print(f.read())
def check_decorators(): binary_result = "binary_result.out" mpi_result = "mpi_result.out" ompss_result = "ompss_result.out" check_binary(binary_result) check_mpi(mpi_result) check_ompss(ompss_result) compss_wait_on_file(binary_result) compss_wait_on_file(mpi_result) compss_wait_on_file(ompss_result) binary_result_fd = compss_open(binary_result) mpi_result_fd = compss_open(mpi_result) ompss_result_fd = compss_open(ompss_result) binary_content = binary_result_fd.readlines() # noqa mpi_content = mpi_result_fd.readlines() # noqa ompss_content = ompss_result_fd.readlines() # noqa binary_result_fd.close() # noqa mpi_result_fd.close() # noqa ompss_result_fd.close() # noqa os.remove(binary_result) os.remove(mpi_result) os.remove(ompss_result) shutil.rmtree(TEMPORARY_DIRECTORY) assert len(binary_content) == 1 assert len(mpi_content) == 1 assert len(ompss_content) == 1 print(binary_content) print(mpi_content) print(ompss_content)
def test_simple(initial_value): file_name = "counter" # Write value with open(file_name, 'w') as fos: fos.write(initial_value) print("Initial counter value is " + initial_value) # Execute increment increment(file_name) # Write new value with compss_open(file_name, 'r+') as fis: final_value = fis.read() print("Final counter value is " + final_value)
def read_in_chunks(file_name, chunk_size=1024): """Lazy function (generator) to read a file piece by piece. Default chunk size: 1k.""" partition = list() f = compss_open(file_name) collected = 0 for line in f: partition.append(line.rstrip("\n")) collected += sys.getsizeof(line) if collected > chunk_size: yield partition partition = [] collected = 0 if partition: yield partition
def test_simple(initial_value): import bad # intentionally placed import error. Do not remove. file_name = "counter" # Write value with open(file_name, 'w') as fos: fos.write(initial_value) print("Initial counter value is " + initial_value) # Execute increment increment(file_name) # Write new value with compss_open(file_name, 'r+') as fis: final_value = fis.read() print("Final counter value is " + final_value)
def testWorkflowFiles(self): """ Test a workflow with FILES """ from pycompss.api.api import compss_open fin = "inwork" fout = "outwork" content = "Before the task " with open(fin, 'w') as f: f.write(content) res = self.fileIn(fin) f2 = self.fileOut(fout, res) f3 = self.fileInOut(fout) with compss_open(fout, 'r') as f: content_r = f.read() content += "\n===> INOUT FILE ADDED CONTENT" self.assertEqual( content, content_r, "strings are not equal: {}, {}".format(content_r, content))
def testFileOUT(self): """ Test FILE_OUT """ from pycompss.api.api import compss_wait_on from pycompss.api.api import compss_open fout = "outfile" content = "OUT FILE CONTENT" res = self.fileOut(fout, content) res = compss_wait_on(res) with compss_open(fout, 'r') as fout_r: content_r = fout_r.read() # The final file is only stored after the execution. # During the execution, you have to use the compss_open, which will # provide the real file where the output file is. # fileInFolder = os.path.exists(fout) # self.assertTrue(fileInFolder, "FILE_OUT is not in the final location") self.assertEqual(res, content, "strings are not equal: {}, {}".format(res, content)) self.assertEqual(content_r, content, "strings are not equal: {}, {}".format(content_r, content))
def testFileINOUT(self): """ Test FILE_INOUT """ from pycompss.api.api import compss_wait_on from pycompss.api.api import compss_open finout = "inoutfile" content = "INOUT FILE CONTENT" with open(finout, 'w') as f: f.write(content) res = self.fileInOut(finout) res = compss_wait_on(res) with compss_open(finout, 'r') as finout_r: content_r = finout_r.read() content += "\n===> INOUT FILE ADDED CONTENT" self.assertEqual(res, content, "strings are not equal: {}, {}".format(res, content)) self.assertEqual(content_r, content, "strings are not equal: {}, {}".format(content_r, content))
def test_basic_types(self): """ Tries primitive types parameter passing. """ print("Running basic types task") filename = "basic_types_file" b_val = True c_val = 'E' s_val = "My Test" by_val = 7 sh_val = 77 i_val = 777 l_val = 7777 f_val = 7.7 d_val = 7.77777 test_basic_types(filename, b_val, c_val, s_val, by_val, sh_val, i_val, l_val, f_val, d_val) with compss_open(filename, "r") as f_channel: line = f_channel.readline() verify_line(line, "TEST BASIC TYPES\n") line = f_channel.readline() verify_line(line, "- boolean: " + str(b_val) + "\n") line = f_channel.readline() verify_line(line, "- char: " + str(c_val) + "\n") line = f_channel.readline() verify_line(line, "- String: " + str(s_val) + "\n") line = f_channel.readline() verify_line(line, "- byte: " + str(by_val) + "\n") line = f_channel.readline() verify_line(line, "- short: " + str(sh_val) + "\n") line = f_channel.readline() verify_line(line, "- int: " + str(i_val) + "\n") line = f_channel.readline() verify_line(line, "- long: " + str(l_val) + "\n") line = f_channel.readline() verify_line(line, "- float: " + str(f_val) + "\n") line = f_channel.readline() verify_line(line, "- double: " + str(d_val) + "\n") line = f_channel.readline() verify_line(line, None) compss_delete_file(filename) compss_barrier() print("\t OK")