def test_compss_call():
context.set_pycompss_context(context.MASTER)
my_compss = COMPSs(app_name="date")
f = my_compss(dummy_function)
result = f()
context.set_pycompss_context(context.OUT_OF_SCOPE)
assert result == 1, "Wrong expected result (should be 1)."
def stop(sync=False):
"""
Runtime stop.
:param sync: Scope variables synchronization [ True | False ]
(default: False)
:return: None
"""
from pycompss.api.api import compss_stop
print("****************************************************")
print("*************** STOPPING PyCOMPSs ******************")
print("****************************************************")
logger = logging.getLogger(__name__)
if sync:
sync_msg = "Synchronizing all future objects left on the user scope."
print(sync_msg)
logger.debug(sync_msg)
from pycompss.api.api import compss_wait_on
ipython = globals()['__builtins__']['get_ipython']()
# import pprint
# pprint.pprint(ipython.__dict__, width=1)
raw_code = ipython.__dict__['user_ns']
for k in raw_code:
obj_k = raw_code[k]
if not k.startswith('_'): # not internal objects
if type(obj_k) == binding.Future:
print("Found a future object: %s" % str(k))
logger.debug("Found a future object: %s" % (k, ))
ipython.__dict__['user_ns'][k] = compss_wait_on(obj_k)
elif obj_k in pending_to_synchronize.values():
print("Found an object to synchronize: %s" % str(k))
logger.debug("Found an object to synchronize: %s" % (k, ))
ipython.__dict__['user_ns'][k] = compss_wait_on(obj_k)
else:
pass
else:
print("Warning: some of the variables used with PyCOMPSs may")
print(" have not been brought to the master.")
if STREAMING:
logger.debug("Stopping streaming")
stop_streaming(logger)
if PERSISTENT_STORAGE:
logger.debug("Stopping persistent storage")
stop_storage()
compss_stop()
__clean_temp_files__()
# Let the Python binding know we are not at master anymore
context.set_pycompss_context(context.OUT_OF_SCOPE)
print("****************************************************")
logger.debug("--- END ---")
def test_multinode_call_worker_with_slurm():
context.set_pycompss_context(context.WORKER)
# prepare test setup
os.environ["COMPSS_NUM_NODES"] = "2"
os.environ["COMPSS_NUM_THREADS"] = "2"
os.environ["COMPSS_HOSTNAMES"] = "hostname1,hostname2"
os.environ["SLURM_NTASKS"] = "2"
os.environ["SLURM_NNODES"] = "2"
os.environ["SLURM_NODELIST"] = "hostname1,hostname2"
os.environ["SLURM_TASKS_PER_NODE"] = "2"
os.environ["SLURM_MEM_PER_NODE"] = "2"
os.environ["SLURM_MEM_PER_CPU"] = "2"
# call
my_multinode = MultiNode()
f = my_multinode(dummy_function)
result = f()
# clean test setup
del os.environ["COMPSS_NUM_NODES"]
del os.environ["COMPSS_NUM_THREADS"]
del os.environ["COMPSS_HOSTNAMES"]
del os.environ["SLURM_NTASKS"]
del os.environ["SLURM_NNODES"]
del os.environ["SLURM_NODELIST"]
del os.environ["SLURM_TASKS_PER_NODE"]
del os.environ["SLURM_MEM_PER_NODE"]
del os.environ["SLURM_MEM_PER_CPU"]
# Check result
assert result == 1, \
"Wrong expected result (should be 1)."
def compss_worker(tracing, task_id, storage_conf, params, log_json):
# type: (bool, str, str, list, str) -> int
""" Worker main method (invoked from __main__).
:param tracing: Tracing boolean
:param task_id: Task identifier
:param storage_conf: Storage configuration file
:param params: Parameters following the common order of the workers
:param log_json: Logger configuration file.
:return: Exit code
"""
if __debug__:
logger = logging.getLogger('pycompss.worker.gat.worker')
logger.debug("Starting Worker")
# Set the binding in worker mode
import pycompss.util.context as context
context.set_pycompss_context(context.WORKER)
result = execute_task("".join(
("Task ", task_id)), storage_conf, params, tracing, logger, log_json,
(), False, dict(), None, None)
# Result contains:
# exit_code, new_types, new_values, timed_out, except_msg = result
exit_code, _, _, _, _ = result
if __debug__:
logger.debug("Finishing Worker")
return exit_code
def test_implement_call_old_mode():
context.set_pycompss_context(context.MASTER)
my_implementation = Implement(sourceClass="s_class", method="s_method")
f = my_implementation(dummy_function)
result = f()
context.set_pycompss_context(context.OUT_OF_SCOPE)
assert result == 1, "Wrong expected result (should be 1)."
def test_multinode_call_master():
context.set_pycompss_context(context.MASTER)
my_multinode = MultiNode()
f = my_multinode(dummy_function)
result = f()
assert result == 1, \
"Wrong expected result (should be 1)."
def test_mpi_binary():
context.set_pycompss_context(context.MASTER)
my_mpi = Mpi(runner=MPI_RUNNER, binary="date", flags="flags")
f = my_mpi(dummy_function)
result = f()
context.set_pycompss_context(context.OUT_OF_SCOPE)
assert result == 1, ERROR_EXPECTED_1
def test_mpi_call():
context.set_pycompss_context(context.MASTER)
my_mpi = Mpi(runner=MPI_RUNNER)
f = my_mpi(dummy_function)
result = f()
context.set_pycompss_context(context.OUT_OF_SCOPE)
assert result == 1, ERROR_EXPECTED_1
def test_mpi_binary():
context.set_pycompss_context(context.MASTER)
my_mpi = MPI(runner="mpirun", binary="date", flags="flags")
f = my_mpi(dummy_function)
result = f()
assert result == 1, \
"Wrong expected result (should be 1)."
def test_opencl_call():
context.set_pycompss_context(context.MASTER)
my_opencl = OpenCL(kernel="date")
f = my_opencl(dummy_function)
result = f()
assert result == 1, \
"Wrong expected result (should be 1)."
def test_mpi_call():
context.set_pycompss_context(context.MASTER)
my_mpi = MPI(runner="mpirun")
f = my_mpi(dummy_function)
result = f()
assert result == 1, \
"Wrong expected result (should be 1)."
def test_who_contextualized():
from pycompss.util.context import set_pycompss_context
from pycompss.util.context import get_who_contextualized
set_pycompss_context(MASTER)
who = get_who_contextualized()
assert who.__name__ == __name__, "ERROR: Wrong who contextualized."
set_pycompss_context(OUT_OF_SCOPE)
def test_ompss_call():
context.set_pycompss_context(context.MASTER)
my_ompss = OmpSs(binary="date")
f = my_ompss(dummy_function)
result = f()
assert result == 1, \
"Wrong expected result (should be 1)."
def test_io_call():
context.set_pycompss_context(context.MASTER)
my_io = IO()
f = my_io(dummy_function)
result = f()
assert result == 1, \
"Wrong expected result (should be 1)."
def test_decorator_instantiation():
context.set_pycompss_context(context.MASTER)
my_deco = PyCOMPSsDecorator(decorator_name=DECORATOR_NAME)
context.set_pycompss_context(context.OUT_OF_SCOPE)
assert (
my_deco.decorator_name == DECORATOR_NAME
), "The decorator name must be @decorator."
def test_opencl_instantiation():
context.set_pycompss_context(context.MASTER)
my_opencl = OpenCL(kernel="date")
context.set_pycompss_context(context.OUT_OF_SCOPE)
assert (
my_opencl.decorator_name == "@opencl"
), "The decorator name must be @opencl."
def test_constraint_call():
context.set_pycompss_context(context.MASTER)
my_constraint = Constraint()
f = my_constraint(dummy_function)
result = f()
context.set_pycompss_context(context.OUT_OF_SCOPE)
assert result == 1, "Wrong expected result (should be 1)."
def compss_worker(tracing, task_id, storage_conf, params):
"""
Worker main method (invocated from __main__).
:param tracing: Tracing boolean
:param task_id: Task identifier
:param storage_conf: Storage configuration file
:param params: Parameters following the common order of the workers
:return: Exit code
"""
if __debug__:
logger = logging.getLogger('pycompss.worker.gat.worker')
logger.debug("Starting Worker")
# Set the binding in worker mode
import pycompss.util.context as context
context.set_pycompss_context(context.WORKER)
result = execute_task("Task " + task_id, storage_conf, params, tracing,
logger, None)
exit_code, new_types, new_values, timed_out, except_msg = result
if __debug__:
logger.debug("Finishing Worker")
return exit_code
def test_on_failure_call():
context.set_pycompss_context(context.MASTER)
my_on_failure = on_failure(management="IGNORE")
f = my_on_failure(dummy_function)
result = f()
context.set_pycompss_context(context.OUT_OF_SCOPE)
assert result == 1, "Wrong expected result (should be 1)."
def test_container_call():
context.set_pycompss_context(context.MASTER)
my_bin = Container(engine="docker", image="dummy")
f = my_bin(dummy_function)
result = f()
context.set_pycompss_context(context.OUT_OF_SCOPE)
assert result == 1, "Wrong expected result (should be 1)."
def test_in_worker_context():
from pycompss.util.context import in_worker
from pycompss.util.context import set_pycompss_context
set_pycompss_context(WORKER)
worker_context = in_worker()
assert worker_context is True, WORKER_CONTEXT_ERROR
set_pycompss_context(OUT_OF_SCOPE)
def test_local_instantiation():
if using_mypy:
raise Exception("UNSUPPORTED WITH MYPY")
context.set_pycompss_context(context.MASTER)
result = dummy_function(1, 2)
context.set_pycompss_context(context.OUT_OF_SCOPE)
assert result == 3, "Wrong expected result (should be 3)."
def test_binary_call():
context.set_pycompss_context(context.MASTER)
my_bin = Binary(binary="date")
f = my_bin(dummy_function)
result = f()
context.set_pycompss_context(context.OUT_OF_SCOPE)
assert result == 1, "Wrong expected result (should be 1)."
def test_in_master_context():
from pycompss.util.context import in_master
from pycompss.util.context import set_pycompss_context
set_pycompss_context(MASTER)
master_context = in_master()
assert master_context is True, MASTER_CONTEXT_ERROR
set_pycompss_context(OUT_OF_SCOPE)
def test_decaf_call():
context.set_pycompss_context(context.MASTER)
my_decaf = Decaf(df_script="date")
f = my_decaf(dummy_function)
result = f()
assert result == 1, \
"Wrong expected result (should be 1)."
def test_decorator_resolve_workingDir(): # noqa
context.set_pycompss_context(context.MASTER)
working_dir = "/tmp"
my_deco = PyCOMPSsDecorator(decorator_name="@decorator",
workingDir=working_dir)
my_deco.__resolve_working_dir__()
assert my_deco.kwargs['working_dir'] == working_dir, \
"Wrong working directory."
def test_binary_existing_core_element():
context.set_pycompss_context(context.MASTER)
my_bin = Binary(binary="date")
f = my_bin(dummy_function)
# a higher level decorator would place the compss core element as follows:
_ = f(compss_core_element=CE())
assert (CORE_ELEMENT_KEY not in my_bin.kwargs
), "Core Element is not defined in kwargs dictionary."
def test_who_contextualized():
from pycompss.util.context import set_pycompss_context
from pycompss.util.context import get_who_contextualized
set_pycompss_context(MASTER)
who = get_who_contextualized()
assert __name__ in who or "nose.case" in who, \
"ERROR: Wrong who (%s) contextualized." % str(who)
set_pycompss_context(OUT_OF_SCOPE)
def test_decaf_existing_core_element():
context.set_pycompss_context(context.MASTER)
my_decaf = Decaf(df_script="date")
f = my_decaf(dummy_function)
# a higher level decorator would place the compss core element as follows:
_ = f(compss_core_element=CE())
assert CORE_ELEMENT_KEY not in my_decaf.kwargs, \
"Core Element is not defined in kwargs dictionary."
def test_decorator_resolve_fail_by_exit_value_str():
context.set_pycompss_context(context.MASTER)
fail_by_exit_value = "true"
my_deco = PyCOMPSsDecorator(decorator_name="@decorator",
fail_by_exit_value=fail_by_exit_value)
my_deco.__resolve_fail_by_exit_value__()
assert my_deco.kwargs['fail_by_exit_value'] == fail_by_exit_value, \
"Wrong fail_by_exit_value."
