def test_per_worker_score():
file1 = STORAGE_PATH + "file1.txt"
file2 = STORAGE_PATH + "file2.txt"
createNewFile(file1)
print("Created new file 1")
createNewFile(file2)
print("Created new file 2")
range1 = 8
for i in range(range1):
increment1(i, file1)
increment2(i, file2)
file1 = compss_wait_on(file1)
file2 = compss_wait_on(file2)
for i in range(range1):
increment1(i, file1)
increment2(i, file2)
with compss_open(file1, 'r') as fis:
final_value1 = fis.read()
with compss_open(file2, 'r') as fis:
final_value2 = fis.read()
#compss_wait_on(result1)
#compss_wait_on(result2)
print("Result 1 = " + str(final_value1))
print("Result 2 = " + str(final_value2))
def check_decorators():
binary_result = "binary_result.out"
mpi_result = "mpi_result.out"
ompss_result = "ompss_result.out"
check_binary(binary_result)
check_mpi(mpi_result)
check_ompss(ompss_result)
compss_wait_on_file(binary_result)
compss_wait_on_file(mpi_result)
compss_wait_on_file(ompss_result)
binary_result_fd = compss_open(binary_result)
mpi_result_fd = compss_open(mpi_result)
ompss_result_fd = compss_open(ompss_result)
binary_content = binary_result_fd.readlines()
mpi_content = mpi_result_fd.readlines()
ompss_content = ompss_result_fd.readlines()
binary_result_fd.close()
mpi_result_fd.close()
ompss_result_fd.close()
os.remove(binary_result)
os.remove(mpi_result)
os.remove(ompss_result)
assert len(binary_content) == 1
assert len(mpi_content) == 1
assert len(ompss_content) == 1
print(binary_content)
print(mpi_content)
print(ompss_content)
def testMultiReturnFile(self):
""" Test multireturn FILE """
fretout1 = "retFile1"
fretout2 = "retFile2"
content1 = "RETURN FILE CONTENTS A"
content2 = "RETURN FILE CONTENTS B"
retResA, retResB = self.multireturnFile(fretout1, fretout2, content1, content2)
retResA = compss_wait_on(retResA)
retResB = compss_wait_on(retResB)
with compss_open(retResA, 'r') as f:
content1_r = f.read()
with compss_open(retResB, 'r') as f:
content2_r = f.read()
self.assertEqual(res, fret, "strings are not equal: {}, {}".format(content2_r, content2))
self.assertEqual(content_r, content, "strings are not equal: {}, {}".format(content1_r, content1))
def load_file(self, file_path, chunk_size=1024, worker_read=False):
"""
Read file in chunks and save it onto partitions.
:param file_path: a path to a file to be loaded
:param chunk_size: size of chunks in bytes
:param worker_read:
:return:
>>> with open("test.file", "w") as testFile:
... _ = testFile.write("Hello world!")
>>> DDS().load_file("test.file", 6).collect()
['Hello ', 'world!']
"""
if worker_read:
fp = open(file_path)
fp.seek(0, 2)
total = fp.tell()
parsed = 0
while parsed < total:
self.partitions.append(
load_partition_from_file(file_path, parsed, chunk_size))
parsed += chunk_size
else:
f = compss_open(file_path, 'r')
chunk = f.read(chunk_size)
while chunk:
self.partitions.append(task_load(chunk))
chunk = f.read(chunk_size)
return self
def test_simple(self):
from pycompss.api.api import compss_open
initial_value = '1'
file_name = "counter"
# Write value
fos = open(file_name, 'w')
fos.write(initial_value)
fos.close()
print("Initial counter value is " + initial_value)
# Execute increment
increment(file_name)
# Write new value
fis = compss_open(file_name, 'r+')
final_value = fis.read()
fis.close()
print("Final counter value is " + final_value)
expected_final_value = '2'
if final_value != expected_final_value:
print("Simple increment test result is incorrect. "
"Expected: %s, got: %s" %
(expected_final_value, final_value))
self.assertEqual(final_value, expected_final_value)
def test_mp_file_access():
print "test_file_mp_access"
print "test_function_files"
fin = 'infile'
finout = 'inoutfile'
fout = 'outfile'
fin_d = open(fin, 'w')
finout_d = open(finout, 'w')
fin_d.write('IN FILE CONTENT')
finout_d.write('INOUT FILE INITIAL CONTENT')
fin_d.close()
finout_d.close()
function_files(fin, finout, fout)
fout_d = compss_open(fout, 'r+')
if fout_d.read() == 'OUT FILE CONTENT':
print "- File access from MP: OK"
else:
print "- File access from MP: ERROR"
fout_d.write("\n===> OUT FILE ADDED CONTENT")
fout_d.close()
function_files(fout, finout, fin)
def testFileIN_OUTDuplic(self):
""" Test FILE_OUT """
from pycompss.api.api import compss_wait_on
from pycompss.api.api import compss_open
if not os.path.exists("id1"):
os.mkdir("id1")
if not os.path.exists("id2"):
os.mkdir("id2")
fin = "id1/outfile"
fout = "id2/outfile"
content_1 = "IN FILE CONTENT"
content_2 = "OUT FILE CONTENT"
self.fileOut(fin, content_1)
res1, res2 = self.fileInOutDuplicated(fin, fout, "outfile", content_2)
res1 = compss_wait_on(res1)
with compss_open(fout, 'r') as fout_r:
content_r = fout_r.read()
# The final file is only stored after the execution.
# During the execution, you have to use the compss_open, which will
# provide the real file where the output file is.
# fileInFolder = os.path.exists(fout)
# self.assertTrue(fileInFolder, "FILE_OUT is not in the final location")
self.assertEqual(
res1, content_1,
"strings are not equal: {}, {}".format(res1, content_1))
self.assertEqual(
content_r, content_2,
"strings are not equal: {}, {}".format(content_r, content_2))
def test_default_file_out(self):
initial_file = "my_out_file.txt"
i_will_fail_file_out(initial_file)
with compss_open(initial_file) as f:
content = f.read()
assert content == "EMPTY FILE OUT", "ERROR: Wrong file inout (%s != EMPTY FILE OUT)" % content
compss_delete_file(initial_file)
def test_mp_file_access():
print("test_file_mp_access")
print("test_function_files")
fin = 'infile'
finout = 'inoutfile'
fout = 'outfile'
fin_d = open(fin, 'w')
finout_d = open(finout, 'w')
fin_d.write('IN FILE CONTENT')
finout_d.write('INOUT FILE INITIAL CONTENT')
fin_d.close()
finout_d.close()
function_files(fin, finout, fout)
fout_d = compss_open(fout, 'r+')
if fout_d.read() == 'OUT FILE CONTENT':
print("- File access from MP: OK")
else:
print("- File access from MP: ERROR")
fout_d.write("\n===> OUT FILE ADDED CONTENT")
fout_d.close()
function_files(fout, finout, fin)
def testFileIN_INOUTDuplic(self):
""" Test FILE_INOUT """
from pycompss.api.api import compss_wait_on
from pycompss.api.api import compss_open
if not os.path.exists("id1"):
os.mkdir("id1")
if not os.path.exists("id2"):
os.mkdir("id2")
fin = "id1/inoutfile"
finout = "id2/inoutfile"
content1 = "INOUT FILE CONTENT 1"
content2 = "INOUT FILE CONTENT 2"
self.fileOut(fin, content1)
self.fileOut(finout, content2)
res1, res2 = self.fileInInoutDpulicated(fin, finout, "inoutfile")
res1 = compss_wait_on(res1)
with compss_open(finout, 'r') as finout_r:
content_r = finout_r.read()
content2 += "\n===> INOUT FILE ADDED CONTENT"
self.assertEqual(
res1, content1,
"strings 1 are not equal: {}, {}".format(res1, content1))
self.assertEqual(
content_r, content2,
"strings 2 are not equal: {}, {}".format(content_r, content2))
def testDeleteFile(self):
# Check and get parameters
initial_value = '1'
counter_name = 'counter_INOUT' # check that this file does not exist after the execution
counter_name_IN = 'counter_IN' # check that this file does not exist after the execution
counter_name_OUT = 'counter_OUT' # check that this file does not exist after the execution
for i in range(3):
# Write value
if i <= 1:
fos = open(counter_name, 'w')
fos.write(initial_value)
fos.close()
fos2 = open(counter_name_IN, 'w')
fos2.write(initial_value)
fos2.close()
print(('Initial counter value is %s' % initial_value))
# Execute increment
increment(counter_name)
increment2(counter_name_IN, counter_name_OUT)
# Read new value
print('After sending task')
if i == 0:
compss_delete_file(counter_name)
compss_delete_file(counter_name_IN)
compss_delete_file(counter_name_OUT)
fis = compss_open(counter_name, 'r+')
final_value = fis.read()
fis.close()
print(('Final counter value is %s' % final_value))
self.assertEqual(final_value, '3')
compss_delete_file(counter_name)
def main_complex_example():
# Imports
from pycompss.api.api import compss_open
# LAUNCH COMPLEX EXAMPLE
if __debug__:
print("Launching MPI complex example")
# Constants
n = 4
m = 8
block_size = 16
# Generate data
data = [[None for _ in range(m)] for _ in range(n)]
for i in range(n):
for j in range(m):
data[i][j] = "file_" + str(i) + "_" + str(j)
init_data_block_in_file(data[i][j], block_size)
# Process data
compact_data = [None for _ in range(n)]
final_data = "final_data"
for i in range(n):
compact_data[i] = "compact_" + str(i)
mpi_compact(compact_data[i], *data[i])
mpi_compact(final_data, *compact_data)
# Synchronise
with compss_open(final_data, "r") as f:
print(f.read())
# DONE
if __debug__:
print("DONE Launching MPI complex example")
def testFileManagementINOUT(self):
inoutfile = "src/inoutfile"
mySedINOUT('-i', 's/Hi/HELLO/g', inoutfile)
with compss_open(inoutfile, "r") as finout_r:
content_r = finout_r.read()
# Check if there are no Hi words, and instead there is HELLO
if 'Hi' in content_r:
self.fail("INOUT File failed.")
def test_default_file_inout(self):
initial_file = "my_inout_file.txt"
with open(initial_file, 'w') as f:
f.write("INITIAL FILE INOUT")
i_will_fail_file_inout(initial_file)
with compss_open(initial_file) as f:
content = f.read()
assert content == "EMPTY FILE INOUT", "ERROR: Wrong file inout (%s != EMPTY FILE INOUT)" % content
compss_delete_file(initial_file)
def printCounterValue(filePath, pos):
from pycompss.api.api import compss_open
# Read value counter 1
fis = compss_open(filePath, 'r+')
counter = fis.read()
fis.close()
# Print values
print "- Counter" + str(pos) + " value is " + counter
def test_file_dependencies_complex(self):
"""
Creates a file on a task, verifies its content in a second one,
a third task updates its value, and, then, the master checks
the value. After that, the master updates the value, verifies its
content locally and on a task.
Later, it updates the value on a task, checks the value on another
task, and updates twice the value on using two tasks. Finally, the
value returns to the master so it checks it.
"""
print("Testing file dependencies - Complex Version")
file_name = "dependencies_file_2"
create_file_with_content(INITIAL_CONTENT, file_name)
check_file_with_content(INITIAL_CONTENT, file_name)
check_and_update_file_with_content(INITIAL_CONTENT, UPDATED_CONTENT_1, file_name)
with compss_open(file_name, "r") as f_channel:
line = f_channel.readline()
verify_line(line, UPDATED_CONTENT_1)
# Update File Content on Main
with compss_open(file_name, "w") as f_channel:
f_channel.write(UPDATED_CONTENT_2)
# Verify File update on Main
with compss_open(file_name, "r") as f_channel:
line = f_channel.readline()
verify_line(line, UPDATED_CONTENT_2)
check_file_with_content(UPDATED_CONTENT_2, file_name)
check_and_update_file_with_content(UPDATED_CONTENT_2, UPDATED_CONTENT_3, file_name)
check_file_with_content(UPDATED_CONTENT_3, file_name)
check_and_update_file_with_content(UPDATED_CONTENT_3, UPDATED_CONTENT_4, file_name)
check_and_update_file_with_content(UPDATED_CONTENT_4, UPDATED_CONTENT_5, file_name)
with compss_open(file_name, "r") as f_channel:
line = f_channel.readline()
verify_line(line, UPDATED_CONTENT_5)
compss_barrier()
compss_delete_file(file_name)
print("\t OK")
def testCheckFileNames(self):
f = "src/infile"
fp = "src/infile"
name = "infile"
fout = "checkFileNamesResult.txt"
exit_value = checkFileNames(f, fp, name, fout)
exit_value = compss_wait_on(exit_value)
with compss_open(fout) as result:
data = result.read()
print("CheckFileNamesResult: " + str(data))
self.assertEqual(exit_value, 0, "At least one file name is NOT as expected: {}, {}, {}".format(f, fp, name))
def testReturnFile(self):
""" Test return FILE """
from pycompss.api.api import compss_wait_on
fret = "fret"
content = "RETURN FILE CONTENT"
res = self.returnFile(fret, content)
res = compss_wait_on(res)
with compss_open(res, 'r') as f:
content_r = f.read()
self.assertEqual(res, fret, "strings are not equal: {}, {}".format(res, content))
self.assertEqual(content_r, content, "strings are not equal: {}, {}".format(content_r, content))
def test_binary_INOUT(self):
message = 'Hi, this is a test.'
f = "myfile.txt"
create_input(message, f)
mySedINOUT('-i', 's/Hi/HELLO/g', f)
with compss_open(f, "r") as f_result:
content_r = f_result.read()
# Check if there are no Hi words, and instead there is HELLO
if 'Hi' in content_r:
self.fail("INOUT File failed.")
if not 'HELLO' in content_r:
self.fail("INOUT File failed.")
def printCounterValue(filePath, pos):
from pycompss.api.api import compss_open
# Read value counter 1
fis = compss_open(filePath, 'r')
counter = fis.read()
fis.close()
# Print values
print("- Counter" + str(pos) + " value is " + counter)
expected = INITIAL_VALUE + 1
if int(counter) != expected:
print(" -Incorrect counter value " + counter + " expected " + str(expected))
exit(-1)
def test_function_files(self):
""" test function files """
from pycompss.api.api import compss_open
# todo hacer tests separados
fin, finout, fout = 'infile', 'inoutfile', 'outfile'
fin_d = open(fin, 'w')
finout_d = open(finout, 'w')
fin_d.write('IN FILE CONTENT')
finout_d.write('INOUT FILE INITIAL CONTENT')
fin_d.close()
finout_d.close()
self.function_files(fin, finout, fout)
# add check content
fin_d = open(fin, 'r')
self.assertEqual(fin_d.read(), 'IN FILE CONTENT')
finout_d = compss_open(finout, 'r')
self.assertEqual(
finout_d.read(),
'INOUT FILE INITIAL CONTENT\n===> INOUT FILE ADDED CONTENT')
fout_d = compss_open(fout, 'r')
self.assertEqual(fout_d.read(), "OUT FILE CONTENT")
def test_task_to_main(self):
"""
Creates a file on a task and reads it on the master.
"""
print("Creating file on task and using it on main")
file_name = "task_to_main_file"
create_file_with_content(INITIAL_CONTENT, file_name)
with compss_open(file_name, "r") as f_channel:
line = f_channel.readline()
verify_line(line, INITIAL_CONTENT)
line = f_channel.readline()
verify_line(line, None)
compss_barrier()
compss_delete_file(file_name)
print("\t OK")
def test_binary_container(self):
# Imports
from pycompss.api.api import compss_barrier
from pycompss.api.api import compss_wait_on
from pycompss.api.api import compss_open
# Test empty binary execution
task_binary_empty()
compss_barrier()
# Test exit value
ev = task_binary_ev()
ev = compss_wait_on(ev)
self.assertEquals(ev, 0)
# Test working dir
# WARN: Check WD in result script
task_binary_wd()
compss_barrier()
# Test stdout and stderr
stdout = "binary_output.txt"
stderr = "binary_error.txt"
task_binary_std(stdout, stderr)
print("STDOUT:")
with compss_open(stdout, 'r') as f:
content = f.read()
print(content)
self.assertTrue(len(content) != 0)
print("STDERR:")
with compss_open(stderr, 'r') as f:
print(f.read())
def check_decorators():
binary_result = "binary_result.out"
mpi_result = "mpi_result.out"
ompss_result = "ompss_result.out"
check_binary(binary_result)
check_mpi(mpi_result)
check_ompss(ompss_result)
compss_wait_on_file(binary_result)
compss_wait_on_file(mpi_result)
compss_wait_on_file(ompss_result)
binary_result_fd = compss_open(binary_result)
mpi_result_fd = compss_open(mpi_result)
ompss_result_fd = compss_open(ompss_result)
binary_content = binary_result_fd.readlines() # noqa
mpi_content = mpi_result_fd.readlines() # noqa
ompss_content = ompss_result_fd.readlines() # noqa
binary_result_fd.close() # noqa
mpi_result_fd.close() # noqa
ompss_result_fd.close() # noqa
os.remove(binary_result)
os.remove(mpi_result)
os.remove(ompss_result)
shutil.rmtree(TEMPORARY_DIRECTORY)
assert len(binary_content) == 1
assert len(mpi_content) == 1
assert len(ompss_content) == 1
print(binary_content)
print(mpi_content)
print(ompss_content)
def test_simple(initial_value):
file_name = "counter"
# Write value
with open(file_name, 'w') as fos:
fos.write(initial_value)
print("Initial counter value is " + initial_value)
# Execute increment
increment(file_name)
# Write new value
with compss_open(file_name, 'r+') as fis:
final_value = fis.read()
print("Final counter value is " + final_value)
def read_in_chunks(file_name, chunk_size=1024):
"""Lazy function (generator) to read a file piece by piece.
Default chunk size: 1k."""
partition = list()
f = compss_open(file_name)
collected = 0
for line in f:
partition.append(line.rstrip("\n"))
collected += sys.getsizeof(line)
if collected > chunk_size:
yield partition
partition = []
collected = 0
if partition:
yield partition
def test_simple(initial_value):
import bad # intentionally placed import error. Do not remove.
file_name = "counter"
# Write value
with open(file_name, 'w') as fos:
fos.write(initial_value)
print("Initial counter value is " + initial_value)
# Execute increment
increment(file_name)
# Write new value
with compss_open(file_name, 'r+') as fis:
final_value = fis.read()
print("Final counter value is " + final_value)
def testWorkflowFiles(self):
""" Test a workflow with FILES """
from pycompss.api.api import compss_open
fin = "inwork"
fout = "outwork"
content = "Before the task "
with open(fin, 'w') as f:
f.write(content)
res = self.fileIn(fin)
f2 = self.fileOut(fout, res)
f3 = self.fileInOut(fout)
with compss_open(fout, 'r') as f:
content_r = f.read()
content += "\n===> INOUT FILE ADDED CONTENT"
self.assertEqual(
content, content_r,
"strings are not equal: {}, {}".format(content_r, content))
def testFileOUT(self):
""" Test FILE_OUT """
from pycompss.api.api import compss_wait_on
from pycompss.api.api import compss_open
fout = "outfile"
content = "OUT FILE CONTENT"
res = self.fileOut(fout, content)
res = compss_wait_on(res)
with compss_open(fout, 'r') as fout_r:
content_r = fout_r.read()
# The final file is only stored after the execution.
# During the execution, you have to use the compss_open, which will
# provide the real file where the output file is.
# fileInFolder = os.path.exists(fout)
# self.assertTrue(fileInFolder, "FILE_OUT is not in the final location")
self.assertEqual(res, content, "strings are not equal: {}, {}".format(res, content))
self.assertEqual(content_r, content, "strings are not equal: {}, {}".format(content_r, content))
def testFileINOUT(self):
""" Test FILE_INOUT """
from pycompss.api.api import compss_wait_on
from pycompss.api.api import compss_open
finout = "inoutfile"
content = "INOUT FILE CONTENT"
with open(finout, 'w') as f:
f.write(content)
res = self.fileInOut(finout)
res = compss_wait_on(res)
with compss_open(finout, 'r') as finout_r:
content_r = finout_r.read()
content += "\n===> INOUT FILE ADDED CONTENT"
self.assertEqual(res, content, "strings are not equal: {}, {}".format(res, content))
self.assertEqual(content_r, content, "strings are not equal: {}, {}".format(content_r, content))
def test_basic_types(self):
"""
Tries primitive types parameter passing.
"""
print("Running basic types task")
filename = "basic_types_file"
b_val = True
c_val = 'E'
s_val = "My Test"
by_val = 7
sh_val = 77
i_val = 777
l_val = 7777
f_val = 7.7
d_val = 7.77777
test_basic_types(filename, b_val, c_val, s_val, by_val, sh_val, i_val,
l_val, f_val, d_val)
with compss_open(filename, "r") as f_channel:
line = f_channel.readline()
verify_line(line, "TEST BASIC TYPES\n")
line = f_channel.readline()
verify_line(line, "- boolean: " + str(b_val) + "\n")
line = f_channel.readline()
verify_line(line, "- char: " + str(c_val) + "\n")
line = f_channel.readline()
verify_line(line, "- String: " + str(s_val) + "\n")
line = f_channel.readline()
verify_line(line, "- byte: " + str(by_val) + "\n")
line = f_channel.readline()
verify_line(line, "- short: " + str(sh_val) + "\n")
line = f_channel.readline()
verify_line(line, "- int: " + str(i_val) + "\n")
line = f_channel.readline()
verify_line(line, "- long: " + str(l_val) + "\n")
line = f_channel.readline()
verify_line(line, "- float: " + str(f_val) + "\n")
line = f_channel.readline()
verify_line(line, "- double: " + str(d_val) + "\n")
line = f_channel.readline()
verify_line(line, None)
compss_delete_file(filename)
compss_barrier()
print("\t OK")
