def main():
print(" - Parsing arguments")
# Arguments parsing
parser = argparse.ArgumentParser()
parser.add_argument('-a',
'--alpha',
type=int,
help="Alpha parameter",
required=True)
parser.add_argument('-b',
'--beta',
type=int,
help="Beta parameter",
required=True)
parser.add_argument('-g',
'--gamma',
type=int,
help="Gamma parameter",
required=True)
parser.add_argument('-d',
'--delta',
type=str,
help="Delta parameter",
required=True)
parser.add_argument('-e',
'--epsilon',
type=float,
help="Epsilon parameter",
required=True)
parser.add_argument('-z',
'--zeta',
type=int,
help="Zeta parameter",
required=True)
args = parser.parse_args()
alpha = args.alpha
beta = args.beta
gamma = args.gamma
delta = args.delta
epsilon = args.epsilon
zeta = args.zeta
values = [alpha, beta, gamma, delta, epsilon, zeta]
expected_values = [6, 66, 666, "z6666", 6.6, 666666]
for v in values:
if type(v) == int:
assign(v)
compss_barrier()
if values == expected_values:
print("Argparse received the expected arguments and values:\n%s" %
values)
else:
print("Argparse did not received the expected arguments and values.")
print("Expected: %s" % (expected_values))
print("Got: %s" % (values))
def testNumbaNjit(self):
# Launch a first task to heat the worker if this test is run individually
heat = increment(1)
heat = compss_wait_on(heat)
repetitions = 200
# Test without njit
no_njit_start_time = time.time()
for i in range(repetitions):
# out = calcul2(np.ones((1000, 1000)))
out = calcul2(np.arange(1000000).reshape(1000, 1000))
compss_barrier()
no_njit_time = time.time() - no_njit_start_time
last_result = compss_wait_on(out)
# Test njit
njit_start_time = time.time()
for i in range(repetitions):
# out_njit = calcul2_njit(np.ones((1000, 1000)))
out_njit = calcul2_njit(np.arange(1000000).reshape(1000, 1000))
compss_barrier()
njit_time = time.time() - njit_start_time
last_result_njit = compss_wait_on(out_njit)
# Check results
print("NO NJIT TIME: " + str(no_njit_time))
print("NJIT TIME : " + str(njit_time))
self.assertGreater(no_njit_time, njit_time)
# Just check the last result of both tasks to ensure that the
# results are equal
self.assertListEqual(last_result.tolist(), last_result_njit.tolist())
def test_concurrency(self):
"""
Launches PARALLEL_TEST_COUNT independent tasks and validates that the runtime executes
them in parallel without overloading the resources.
"""
print("Testing concurrent executions")
reports = []
for i in range(0, PARALLEL_TEST_COUNT):
reports.append(sleep_task())
compss_barrier()
# Verify the number of tasks able to run in parallel in the process
modifications = []
for report in reports:
report = compss_wait_on(report)
modifications.append(WorkerModification(report.start, 1))
modifications.append(WorkerModification(report.end, -1))
import operator
sorted_modifications = sorted(modifications, key=operator.attrgetter('moment'))
current_count = 0
max_count = 0
for mod in sorted_modifications:
current_count = current_count + mod.modification
max_count = max(max_count, current_count)
if current_count > PARALLEL_TEST_MAX_COUNT:
raise Exception("Scheduling does not properly manage the maximum number of tasks")
if max_count != PARALLEL_TEST_MAX_COUNT:
raise Exception("Worker in master does not hold as many task as possible")
print("\tOK")
def hybrid_optimization():
'''
CLASSICAL SECTION: define optimizator and initialize classical code
'''
# SPSA parameters
a = 0.01
c = 0.001
alpha = 0.602
gamma = 0.101
# iters = 3000
iters = 2
A = 50 / iters
# initial optimization parameters
#N = 7
N = 6
n_params = 4 * N - 1
theta = 0.1 * np.ones(n_params)
theta_max = (np.pi / 2) * np.ones(n_params)
theta_min = (-np.pi / 2) * np.ones(n_params)
# solve opt
opt = spsa.spsa(A, a, c, alpha, gamma, theta, theta_max, theta_min, iters)
compss_barrier()
'''
HYBRID SECTION: spsa.step repetedly calls a function evaluation on the quantum device.
'''
l_p, l_m = opt.step(ising_1D_XX_Z)
print(l_p)
print(l_m)
def testNumbaVectorize(self):
# Launch a first task to heat the worker if this test is run individually
heat = increment(1)
heat = compss_wait_on(heat)
repetitions = 100
# Test vectorize
no_vectorize_start_time = time.time()
for i in range(repetitions):
out = calcul3(np.ones((500000)))
compss_barrier()
no_vectorize_time = time.time() - no_vectorize_start_time
last_result = compss_wait_on(out)
# Test vectorize
vectorize_start_time = time.time()
for i in range(repetitions):
out_vectorize = calcul3_vectorize(np.ones((500000)))
compss_barrier()
vectorize_time = time.time() - vectorize_start_time
last_result_vectorize = compss_wait_on(out_vectorize)
# Check results
print("NO VECTORIZE TIME: " + str(no_vectorize_time))
print("VECTORIZE TIME : " + str(vectorize_time))
self.assertGreater(no_vectorize_time, vectorize_time)
# Just check the last result of both tasks to ensure that the
# results are equal
self.assertListEqual(last_result.tolist(), last_result_vectorize.tolist())
def measure(name, dataset_name, func, *args, **kwargs):
print("==== STARTING ====", name, dataset_name)
compss_barrier()
s_time = time.time()
func(*args, **kwargs)
compss_barrier()
print("==== TIME ==== ", name, dataset_name, time.time() - s_time)
def testNumbaStencil(self):
# Launch a first task to heat the worker if this test is run individually
heat = increment(1)
heat = compss_wait_on(heat)
repetitions = 10
# Test without stencil
no_stencil_start_time = time.time()
for i in range(repetitions):
# out = calcul5(np.arange(1000000).reshape((1000, 1000)))
out = calcul5(np.ones((1000, 1000)))
compss_barrier()
no_stencil_time = time.time() - no_stencil_start_time
last_result = compss_wait_on(out)
# Test with stencil
stencil_start_time = time.time()
for i in range(repetitions):
# out_stencil = calcul5_stencil(np.arange(1000000).reshape((1000, 1000)))
out_stencil = calcul5_stencil(np.ones((1000, 1000)))
compss_barrier()
stencil_time = time.time() - stencil_start_time
last_result_stencil = compss_wait_on(out_stencil)
# Check results
print("NO STENCIL TIME: " + str(no_stencil_time))
print("STENCIL TIME : " + str(stencil_time))
self.assertGreater(no_stencil_time, stencil_time)
# Just check the last result of both tasks to ensure that the
# results are equal
self.assertListEqual(last_result.tolist(), last_result_stencil.tolist())
def kmeans_frag(numV, k, dim, epsilon, maxIterations, numFrag):
from pycompss.api.api import compss_wait_on, compss_barrier
import time
size = int(numV / numFrag)
startTime = time.time()
X = [genFragment(size, dim) for _ in range(numFrag)]
compss_barrier()
print("Points generation Time {} (s)".format(time.time() - startTime))
mu = init_random(dim, k)
oldmu = []
n = 0
startTime = time.time()
while not has_converged(mu, oldmu, epsilon, n, maxIterations):
oldmu = mu
clusters = [cluster_points_partial(X[f], mu, f * size) for f in range(numFrag)]
partialResult = [partial_sum(X[f], clusters[f], f * size) for f in range(numFrag)]
mu = reduce(reduceCentersTask, partialResult)
mu = compss_wait_on(mu)
mu = [mu[c][1] / mu[c][0] for c in mu]
n += 1
print("Kmeans Time {} (s)".format(time.time() - startTime))
return (n, mu)
def cholesky():
import time
import sys
from pycompss.api.api import compss_barrier
MSIZE = int(sys.argv[1])
BSIZE = int(sys.argv[2])
mkl_threads = int(sys.argv[3])
# Generate de matrix
startTime = time.time()
# Generate supermatrix
A = []
genMatrix(A, MSIZE, BSIZE, mkl_threads)
compss_barrier()
initTime = time.time() - startTime
startDecompTime = time.time()
res = cholesky_blocked(A, MSIZE, BSIZE, mkl_threads)
compss_barrier()
decompTime = time.time() - startDecompTime
totalTime = decompTime + initTime
print("---------- PARAMS ----------")
print("MSIZE:{}".format(MSIZE))
print("BSIZE:{}".format(BSIZE))
print("initT:{}".format(initTime))
print("decompT:{}".format(decompTime))
print("totalTime:{}".format(totalTime))
print("----------------------------")
def main():
time.sleep(25)
gen_task1("filename1_1", "filename1_2", "filename1_3", CONTENT)
gen_task1("filename2_1", "filename2_2", "filename2_3", CONTENT)
compss_barrier()
read_task2("filename2_1", "filename1_2", "filename1_3")
read_task1("filename1_1", "filename2_2", "filename2_3")
def measure(name, dataset_name, model, x, y=None):
print("==== STARTING ====", name)
compss_barrier()
s_time = time.time()
model.fit(x, y)
compss_barrier()
print("==== OUTPUT ==== ", dataset_name, time.time() - s_time)
def main_usage_examples():
# Imports
from pycompss.api.api import compss_barrier
from pycompss.api.api import compss_wait_on
from pycompss.api.api import compss_open
import os
# LAUNCH SOME MPI EXAMPLES
if __debug__:
print("Launching MPI usage examples")
# Launch empty MPI
if __debug__:
print("- Launching empty MPI")
mpi_example_empty()
compss_barrier()
# Launch MPI with parameters
if __debug__:
print("- Launching MPI with parameters")
num = 5
string = "hello world"
file_path = os.environ["OUTPUT_DIR"] + "/file.in"
with open(file_path, 'w') as f:
f.write("Hello World!\n")
mpi_example_params(num, string, file_path)
# Launch MPI with exit value
if __debug__:
print("- Launching MPI with EV")
ev = mpi_example_with_ev()
ev = compss_wait_on(ev)
print("MPI EXIT VALUE = " + str(ev))
if ev != 0:
raise Exception("ERROR: MPI binary exited with non-zero value")
# Launch MPI with STDOUT and STDERR
if __debug__:
print("- Launching MPI with STDOUT/STDERR")
stdout = os.environ["OUTPUT_DIR"] + "/mpi_task_example.stdout"
stderr = os.environ["OUTPUT_DIR"] + "/mpi_task_example.stderr"
ev = mpi_example_std(stdout, stderr)
ev = compss_wait_on(ev)
print("MPI EXIT VALUE = " + str(ev))
print("MPI STDOUT:")
with open(stdout) as f:
print(f.read())
print("MPI STDERR:")
with open(stderr) as f:
print(f.read())
if __debug__:
print("- Launching MPI with working_dir")
mpi_example_wd()
compss_barrier()
# DONE
if __debug__:
print("DONE Launching MPI usage examples")
def measure(name, dataset_name, func, *args, **kwargs):
print("==== STARTING ====", name, dataset_name)
compss_barrier()
s_time = time.time()
func(*args, **kwargs)
compss_barrier()
print("==== TIME ==== ", name, dataset_name, time.time() - s_time)
print("In worker_working_dir: ", compss_wait_on(get_worker_working_dir()))
def testTaskArgWarn(self):
task1_task(1)
o = task2_task(2)
p = task3_task(3)
compss_barrier()
o = compss_wait_on(o)
p = compss_wait_on(p)
self.assertEqual(o, p)
def qr():
import time
import sys
from pycompss.api.api import compss_barrier
from pycompss.api.api import compss_wait_on
np.set_printoptions(precision=2)
global MSIZE
global BSIZE
global mkl_threads
MSIZE = int(sys.argv[1])
BSIZE = int(sys.argv[2])
mkl_threads = int(sys.argv[3])
verifyOutput = sys.argv[4] == "False"
startTime = time.time()
# Generate de matrix
m2b = genMatrix(mkl_threads)
compss_barrier()
initTime = time.time() - startTime
startDecompTime = time.time()
(Q, R) = qr_blocked(m2b, mkl_threads)
compss_barrier()
decompTime = time.time() - startDecompTime
totalTime = time.time() - startTime
print("PARAMS:------------------")
print("MSIZE:{}".format(MSIZE))
print("BSIZE:{}".format(BSIZE))
print("initT:{}".format(initTime))
print("decompT:{}".format(decompTime))
print("totalTime:{}".format(totalTime))
if(verifyOutput):
Q = compss_wait_on(Q)
R = compss_wait_on(R)
m2b = compss_wait_on(m2b)
print("Input matrix")
print(joinMatrix(m2b))
print("Q*R")
print(joinMatrix(Q)*joinMatrix(R))
print("Generated R")
print(joinMatrix(R))
print("NumPy R")
print(np.linalg.qr(joinMatrix(m2b))[1])
print("Generated Q")
print(joinMatrix(Q))
print("NumPy Q")
print(np.linalg.qr(joinMatrix(m2b))[0])
def foreach(self, f):
"""
Apply a function to each element of this data set without returning
anything.
:param f: a void function
"""
self.map(f)
# Wait for all the tasks to finish
compss_barrier()
def test_worker_producer_master_consumer_file(self):
"""
"""
print("Worker produces file, master consumes")
file_name = "worker_producer_master_consumer"
create_file_with_content_worker(INITIAL_CONTENT, file_name)
check_file_with_content_master(INITIAL_CONTENT, file_name)
compss_barrier()
compss_delete_file(file_name)
print("\t OK")
def postBarrierReduce():
a = [x for x in range(1,NUM_TASKS+1)]
result = []
for element in a:
print("[Element] " + str(element))
result.append(increment(element))
compss_barrier()
final = myreduction_4(operator.add, "test", result)
final = compss_wait_on(final)
print("[LOG] Result Post-Barrier: " + str(final))
return final
def main():
# Start
print("[INFO] Starting application")
start_time = datetime.now()
# Parse arguments
print("[INFO] Parsing application arguments")
app_args = parse_arguments()
# Create streams
print("[INFO] Creating streams")
sensor_base_dir = app_args.stream_base_dir + "/sensor/"
filter_base_dir = app_args.stream_base_dir + "/filter/"
remove_and_create(app_args.stream_base_dir)
os.makedirs(sensor_base_dir)
os.makedirs(filter_base_dir)
fds_sensor = FileDistroStream(alias="sensor", base_dir=sensor_base_dir)
fds_filtered = FileDistroStream(alias="filtered", base_dir=filter_base_dir)
# Create sensor
print("[INFO] Launching sensor")
sensor(fds_sensor, app_args.sensor_num_files,
app_args.sensor_base_sleep_time, app_args.sensor_sleep_random_range)
# Create filters
print("[INFO] Launching filter nested")
big_filter(fds_sensor, fds_filtered, app_args.batch_size,
app_args.filter_base_sleep_time,
app_args.filter_sleep_random_range)
# Create extract
print("[INFO] Launching extract")
elements = extract_info(fds_filtered, app_args.extract_base_sleep_time,
app_args.extract_sleep_random_range)
# Launch task flow computation
print("[INFO] Launching task flow computation")
# elements = compss_wait_on(elements)
res = task_flow(app_args.tf_depth, app_args.tf_base_sleep_time,
app_args.tf_sleep_random_range, *elements)
# Synchronize
print("[INFO] Syncrhonizing final output")
res = compss_wait_on(res)
print("[INFO] TF ended with status: " + str(res))
compss_barrier()
# End
print("[INFO] DONE")
end_time = datetime.now()
elapsed_time = end_time - start_time
print("[TIME] TOTAL ELAPSED: " + str(elapsed_time.total_seconds()))
def test_dynamic(self):
global cn
cn = 2
dynamic_gv_cn()
cn = 4
dynamic_gv_cn()
compss_barrier()
print ("CORRECTNESS IS CHECKED IN THE RESULT SCRIPT")
def test_master_producer_worker_consumer_file(self):
"""
Creates a file and passes it as an input parameter to a task.
"""
print("Master produces file, worker consumes")
file_name = "master_producer_worker_consumer"
create_file_with_content_master(INITIAL_CONTENT, file_name)
check_file_with_content_worker(INITIAL_CONTENT, file_name)
compss_barrier()
compss_delete_file(file_name)
print("\t OK")
def test_master_producer_worker_consumer_master_updates_object(self):
print(
"Master produces object, several workers consume, master updates, worker reads"
)
stringwrapper = create_object_with_content_master(INITIAL_CONTENT)
for i in range(0, PARALLEL_TEST_COUNT):
check_object_with_content_worker(INITIAL_CONTENT, stringwrapper)
check_and_update_object_with_content(INITIAL_CONTENT,
UPDATED_CONTENT_1, stringwrapper)
check_object_with_content_worker(UPDATED_CONTENT_1, stringwrapper)
compss_barrier()
print("\t OK")
def test_main_to_task(self):
"""
Creates a file and passes it as an input parameter to a task.
"""
print("Creating file on main and using it on task")
file_name = "main_to_task_file"
with open(file_name, "w") as f_channel:
f_channel.write(INITIAL_CONTENT)
check_file_with_content(INITIAL_CONTENT, file_name)
compss_barrier()
compss_delete_file(file_name)
print("\t OK")
def test_master_producer_worker_consumer_master_updates_file(self):
"""
"""
print("Master produces file, several workers consume, master updates, worker reads")
file_name = "produce_consume_update"
create_file_with_content_master(INITIAL_CONTENT, file_name)
for i in range(0, PARALLEL_TEST_COUNT):
check_file_with_content_worker(INITIAL_CONTENT, file_name)
check_and_update_file_with_content(INITIAL_CONTENT, UPDATED_CONTENT_1, file_name)
check_file_with_content_worker(UPDATED_CONTENT_1, file_name)
compss_barrier()
print("\t OK")
def dot(A, B, C, set_barrier=False):
'''A COMPSs-PSCO blocked matmul algorithm
A and B (blocks) are PSCOs, while C (blocks) are objects
'''
n, m = len(A), len(B[0])
# as many rows as A, as many columns as B
for i in range(n):
for j in range(m):
for k in range(n):
multiply(A[i][k], B[k][j], C[i][j])
if set_barrier:
from pycompss.api.api import compss_barrier
compss_barrier()
def main():
import sys
import time
from dataclay.api import init, finish
from dataclay.exceptions.exceptions import DataClayException
mqtt_wait = False
if len(sys.argv) == 2:
mqtt_wait = (sys.argv[1] != "False")
init()
from CityNS.classes import DKB, ListOfObjects
# Register MQTT client to subscribe to MQTT server in 192.168.7.42
if mqtt_wait:
client = register_mqtt()
client.loop_start()
# initialize all computing units in all workers
num_cus = 8
for i in range(num_cus):
init_task()
compss_barrier()
# Publish to the MQTT broker that the execution has started
if mqtt_wait:
publish_mqtt(client)
try:
kb = DKB.get_by_alias("DKB")
except DataClayException:
kb = DKB()
list_objects = ListOfObjects()
list_objects.make_persistent()
kb.list_objects = list_objects
kb.make_persistent("DKB")
start_time = time.time()
# execute_trackers(["192.168.50.103"], kb)
execute_trackers([("/tmp/pipe_yolo2COMPSs", "/tmp/pipe_COMPSs2yolo")], kb)
# pipe_paths = [("/tmp/pipe_yolo2COMPSs", "/tmp/pipe_COMPSs2yolo"), ("/tmp/pipe_write", "/tmp/pipe_read")]
# print("ExecTime: " + str(time.time() - start_time))
# print("ExecTime per Iteration: " + str((time.time() - start_time) / NUM_ITERS))
if mqtt_wait:
while CD_PROC < NUM_ITERS:
pass
print("Exiting Application...")
finish()
def test_task_to_main(self):
"""
Creates a file on a task and reads it on the master.
"""
print("Creating file on task and using it on main")
file_name = "task_to_main_file"
create_file_with_content(INITIAL_CONTENT, file_name)
with compss_open(file_name, "r") as f_channel:
line = f_channel.readline()
verify_line(line, INITIAL_CONTENT)
line = f_channel.readline()
verify_line(line, None)
compss_barrier()
compss_delete_file(file_name)
print("\t OK")
def test_object_dependencies(self):
"""
Creates a Stringwrapper on a task, verifies its content in a second one,
a third task updates its value, and, finally, the master checks
the value.
"""
print("Testing object dependencies")
stringwrapper = create_object_with_content(INITIAL_CONTENT)
check_object_with_content(INITIAL_CONTENT, stringwrapper)
check_and_update_object_with_content(INITIAL_CONTENT, UPDATED_CONTENT_1, stringwrapper)
stringwrapper = compss_wait_on(stringwrapper)
line = stringwrapper.value
verify_line(line, UPDATED_CONTENT_1)
compss_barrier()
print("\t OK")
def main():
from pycompss.api.api import compss_barrier, compss_wait_on
tmp_dir = tempfile.mkdtemp(dir="/tmp/")
data = task1(tmp_dir)
task2(data, tmp_dir)
dir_len = io_task(data, tmp_dir)
dir_len = compss_wait_on(dir_len)
compss_barrier()
shutil.rmtree(tmp_dir)
#if dir_len = 2, this means that io_task() did not overlap with task2()
assert dir_len == 1, "IO task did not overlap with the compute task!"
def foreach(self, f):
"""
Apply a function to each element of this data set without returning
anything.
:param f: a void function
:return: null
>>> def dummy(x): print(x)
>>> DDS([1, 2], 2).foreach(dummy)
1
2
"""
self.map(f)
# Wait for all the tasks to finish
compss_barrier()
return
