def __configure_core_element__(self, kwargs, user_function):
# type: (dict, ...) -> None
""" Include the registering info related to @multinode.
IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY].
:param kwargs: Keyword arguments received from call.
:param user_function: Decorated function.
:return: None
"""
if __debug__:
logger.debug("Configuring @multinode core element.")
# Resolve @multinode specific parameters
impl_type = "MULTI_NODE"
if CORE_ELEMENT_KEY in kwargs:
# Core element has already been created in a higher level decorator
# (e.g. @constraint)
kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type)
else:
# @binary is in the top of the decorators stack.
# Instantiate a new core element object, update it and include
# it into kwarg
core_element = CE()
core_element.set_impl_type(impl_type)
kwargs[CORE_ELEMENT_KEY] = core_element
# Set as configured
self.core_element_configured = True
def __configure_core_element__(self, kwargs):
# type: (dict) -> None
""" Include the registering info related to @constraint.
IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY].
:param kwargs: Current keyword arguments to be updated with the core
element information.
:return: None
"""
if __debug__:
logger.debug("Configuring @constraint core element.")
if CORE_ELEMENT_KEY in kwargs:
# Core element has already been created in a higher level decorator
# (e.g. @implements and @compss)
kwargs[CORE_ELEMENT_KEY].set_impl_constraints(self.kwargs)
else:
# @constraint is in the top of the decorators stack.
# Instantiate a new core element object, update it and include
# it into kwarg
core_element = CE()
core_element.set_impl_constraints(self.kwargs)
kwargs[CORE_ELEMENT_KEY] = core_element
# Set as configured
self.core_element_configured = True
def __configure_core_element__(self, kwargs): # noqa
# type: (dict) -> None
""" Include the registering info related to @IO.
IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY].
:param kwargs: Keyword arguments received from call.
:return: None
"""
if __debug__:
logger.debug("Configuring @IO core element.")
# Resolve @io specific parameters
if CORE_ELEMENT_KEY in kwargs:
# Core element has already been created in a higher level decorator
# (e.g. @constraint)
kwargs[CORE_ELEMENT_KEY].set_impl_io(True)
else:
# @binary is in the top of the decorators stack.
# Instantiate a new core element object, update it and include
# it into kwarg
core_element = CE()
core_element.set_impl_io(True)
kwargs[CORE_ELEMENT_KEY] = core_element
def test_multinode_existing_core_element():
context.set_pycompss_context(context.MASTER)
my_multinode = MultiNode()
f = my_multinode(dummy_function)
# a higher level decorator would place the compss core element as follows:
_ = f(compss_core_element=CE())
assert CORE_ELEMENT_KEY not in my_multinode.kwargs, \
"Core Element is not defined in kwargs dictionary."
def test_decaf_existing_core_element():
context.set_pycompss_context(context.MASTER)
my_decaf = Decaf(df_script="date")
f = my_decaf(dummy_function)
# a higher level decorator would place the compss core element as follows:
_ = f(compss_core_element=CE())
assert CORE_ELEMENT_KEY not in my_decaf.kwargs, \
"Core Element is not defined in kwargs dictionary."
def software_f(*args, **kwargs):
# type: (*typing.Any, **typing.Any) -> typing.Any
if not self.scope or not context.in_master():
# Execute the software as with PyCOMPSs so that sequential
# execution performs as parallel.
# To disable: raise Exception(not_in_pycompss(BINARY))
return user_function(*args, **kwargs)
if __debug__:
logger.debug("Executing software_f wrapper.")
if self.constraints is not None:
core_element = CE()
core_element.set_impl_constraints(self.constraints)
kwargs[CORE_ELEMENT_KEY] = core_element
if self.container is not None:
_func = str(user_function.__name__)
impl_type = IMPL_CONTAINER
impl_signature = '.'.join((impl_type, _func))
ce = kwargs.get(CORE_ELEMENT_KEY, CE())
impl_args = [self.container[ENGINE], # engine
self.container[IMAGE], # image
UNASSIGNED, # internal_type
UNASSIGNED, # internal_binary
UNASSIGNED, # internal_func
UNASSIGNED, # working_dir
UNASSIGNED] # fail_by_ev
ce.set_impl_type(impl_type)
ce.set_impl_signature(impl_signature)
ce.set_impl_type_args(impl_args)
kwargs[CORE_ELEMENT_KEY] = ce
if self.decor:
decorator = self.decor
def decor_f():
def f():
ret = decorator(**self.config_args)
return ret(user_function)(*args, **kwargs)
return f()
return decor_f()
else:
# It's a PyCOMPSs task with only @task and @software decorators
return user_function(*args, **kwargs)
def test_binary_existing_core_element():
context.set_pycompss_context(context.MASTER)
my_bin = Binary(binary="date")
f = my_bin(dummy_function)
# a higher level decorator would place the compss core element as follows:
_ = f(compss_core_element=CE())
assert (CORE_ELEMENT_KEY not in my_bin.kwargs
), "Core Element is not defined in kwargs dictionary."
def test_container_existing_core_element():
context.set_pycompss_context(context.MASTER)
my_bin = Container(engine="docker", image="dummy")
f = my_bin(dummy_function)
# a higher level decorator would place the compss core element as follows:
_ = f(compss_core_element=CE())
assert CORE_ELEMENT_KEY not in my_bin.kwargs, \
"Core Element is not defined in kwargs dictionary."
def test_constraint_existing_core_element():
context.set_pycompss_context(context.MASTER)
my_constraint = Constraint()
f = my_constraint(dummy_function)
# a higher level decorator would place the compss core element as follows:
_ = f(compss_core_element=CE())
context.set_pycompss_context(context.OUT_OF_SCOPE)
assert (CORE_ELEMENT_KEY not in my_constraint.kwargs
), "Core Element is not defined in kwargs dictionary."
def __configure_core_element__(self, kwargs, user_function):
# type: (dict, str) -> None
""" Include the registering info related to @container.
IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY].
:param kwargs: Keyword arguments received from call.
:param user_function: Decorated function.
:return: None
"""
if __debug__:
logger.debug("Configuring @container core element.")
# Resolve @container (mandatory) specific parameters
_engine = self.kwargs['engine']
_image = self.kwargs['image']
_func = str(user_function.__name__)
# Type and signature
impl_type = "CONTAINER"
impl_signature = '.'.join((impl_type, _func))
unassigned = "[unassigned]"
impl_args = [
_engine, # engine
_image, # image
unassigned, # internal_type
unassigned, # internal_binary
unassigned, # internal_func
unassigned, # working_dir
unassigned
] # fail_by_ev
if CORE_ELEMENT_KEY in kwargs:
# Core element has already been created in a higher level decorator
# (e.g. @constraint)
kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type)
kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature)
kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args)
else:
# @container is in the top of the decorators stack.
# Instantiate a new core element object, update it and include
# it into kwarg
core_element = CE()
core_element.set_impl_type(impl_type)
core_element.set_impl_signature(impl_signature)
core_element.set_impl_type_args(impl_args)
kwargs[CORE_ELEMENT_KEY] = core_element
# Set as configured
self.core_element_configured = True
def test_implement_existing_core_element():
context.set_pycompss_context(context.MASTER)
my_implementation = Implement(source_class="s_class", method="s_method")
# Hack to mimic registered
my_implementation.first_register = True
f = my_implementation(dummy_function)
# a higher level decorator would place the compss core element as follows:
_ = f(compss_core_element=CE())
context.set_pycompss_context(context.OUT_OF_SCOPE)
assert (CORE_ELEMENT_KEY not in my_implementation.kwargs
), "Core Element is not defined in kwargs dictionary."
def __configure_core_element__(self, kwargs):
# type: (dict) -> None
""" Include the registering info related to @mpmd_mpi.
IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY].
:param kwargs: Keyword arguments received from call.
:return: None
"""
if __debug__:
logger.debug("Configuring @mpmd_mpi core element.")
# Resolve @mpmd_mpi specific parameters
impl_type = "MPMDMPI"
runner = self.kwargs[RUNNER]
# Resolve the working directory
resolve_working_dir(self.kwargs)
# Resolve the fail by exit value
resolve_fail_by_exit_value(self.kwargs)
ppn = str(self.kwargs.get(PROCESSES_PER_NODE, 1))
impl_signature = '.'.join((impl_type, str(ppn)))
prog_params = self.__get_programs_params__()
impl_args = [
runner, self.kwargs[WORKING_DIR], ppn,
self.kwargs[FAIL_BY_EXIT_VALUE]
]
impl_args.extend(prog_params)
if CORE_ELEMENT_KEY in kwargs:
kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type)
kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature)
kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args)
else:
core_element = CE()
core_element.set_impl_type(impl_type)
core_element.set_impl_signature(impl_signature)
core_element.set_impl_type_args(impl_args)
kwargs[CORE_ELEMENT_KEY] = core_element
# Set as configured
self.core_element_configured = True
def __configure_core_element__(self, kwargs):
# type: (dict) -> None
""" Include the registering info related to @ompss.
IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY].
:param kwargs: Keyword arguments received from call.
:return: None
"""
if __debug__:
logger.debug("Configuring @ompss core element.")
# Resolve @ompss specific parameters
binary = self.kwargs['binary']
# Resolve the working directory
self.__resolve_working_dir__()
# Resolve the fail by exit value
self.__resolve_fail_by_exit_value__()
impl_type = "OMPSS"
impl_signature = "".join(("OMPSS.", binary))
impl_args = [
binary, self.kwargs['working_dir'],
self.kwargs['fail_by_exit_value']
]
if CORE_ELEMENT_KEY in kwargs:
# Core element has already been created in a higher level decorator
# (e.g. @constraint)
kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type)
kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature)
kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args)
else:
# @binary is in the top of the decorators stack.
# Instantiate a new core element object, update it and include
# it into kwarg
core_element = CE()
core_element.set_impl_type(impl_type)
core_element.set_impl_signature(impl_signature)
core_element.set_impl_type_args(impl_args)
kwargs[CORE_ELEMENT_KEY] = core_element
# Set as configured
self.core_element_configured = True
def __configure_core_element__(self, kwargs, user_function):
# type: (dict, ...) -> None
""" Include the registering info related to @opencl.
IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY].
:param kwargs: Keyword arguments received from call.
:param user_function: Decorated function.
:return: None
"""
if __debug__:
logger.debug("Configuring @opencl core element.")
# Resolve @opencl specific parameters
kernel = self.kwargs['kernel']
# Resolve the working directory
self.__resolve_working_dir__()
impl_type = 'OPENCL'
impl_signature = '.'.join((impl_type, kernel))
impl_args = [kernel, self.kwargs['working_dir']]
if CORE_ELEMENT_KEY in kwargs:
# Core element has already been created in a higher level decorator
# (e.g. @constraint)
kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type)
kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature)
kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args)
else:
# @binary is in the top of the decorators stack.
# Instantiate a new core element object, update it and include
# it into kwarg
core_element = CE()
core_element.set_impl_type(impl_type)
core_element.set_impl_signature(impl_signature)
core_element.set_impl_type_args(impl_args)
kwargs[CORE_ELEMENT_KEY] = core_element
# Set as configured
self.core_element_configured = True
def __configure_core_element__(self, kwargs):
# type: (dict) -> None
""" Include the registering info related to @on_failure.
IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY].
:param kwargs: Current keyword arguments to be updated with the core
element information.
:return: None
"""
if __debug__:
logger.debug("Configuring @on_failure core element.")
if CORE_ELEMENT_KEY not in kwargs:
# @on_failure is in the top of the decorators stack.
# Instantiate a new core element object, update it and include
# it into kwarg
kwargs[CORE_ELEMENT_KEY] = CE()
# Set as configured
self.core_element_configured = True
def __configure_core_element__(self, kwargs, user_function):
# type: (dict, ...) -> None
""" Include the registering info related to @implement.
IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY].
:param kwargs: Keyword arguments received from call.
:param user_function: Decorated function.
:return: None
"""
if __debug__:
logger.debug("Configuring @implement core element.")
# Resolve @implement specific parameters
if 'sourceClass' in self.kwargs:
another_class = self.kwargs['sourceClass']
self.kwargs['source_class'] = self.kwargs.pop('sourceClass')
else:
another_class = self.kwargs['source_class']
another_method = self.kwargs['method']
ce_signature = '.'.join((another_class, another_method))
impl_type = "METHOD"
# impl_args = [another_class, another_method] - set by @task
if CORE_ELEMENT_KEY in kwargs:
# Core element has already been created in a higher level decorator
# (e.g. @constraint)
kwargs[CORE_ELEMENT_KEY].set_ce_signature(ce_signature)
kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type)
# @task sets the implementation type arguments
# kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args)
else:
# @implement is in the top of the decorators stack.
# Instantiate a new core element object, update it and include
# it into kwarg
core_element = CE()
core_element.set_ce_signature(ce_signature)
core_element.set_impl_type(impl_type)
# @task sets the implementation type arguments
# core_element.set_impl_type_args(impl_args)
kwargs[CORE_ELEMENT_KEY] = core_element
# Set as configured
self.core_element_configured = True
def __configure_core_element__(self, kwargs):
# type: (dict) -> None
""" Include the registering info related to @http.
IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY].
:param kwargs: Keyword arguments received from call.
:return: None
"""
if __debug__:
logger.debug("Configuring @http core element.")
impl_type = "HTTP"
impl_args = [
self.kwargs["service_name"], self.kwargs["resource"],
self.kwargs["request"],
self.kwargs.get("payload", '#'),
self.kwargs.get("payload_type", "application/json"),
self.kwargs.get("produces", '#'),
self.kwargs.get("updates", '#')
]
if CORE_ELEMENT_KEY in kwargs:
# Core element has already been created in a higher level decorator
# (e.g. @constraint)
kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type)
kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args)
else:
# @binary is in the top of the decorators stack.
# Instantiate a new core element object, update it and include
# it into kwarg
core_element = CE()
core_element.set_impl_type(impl_type)
core_element.set_impl_type_args(impl_args)
kwargs[CORE_ELEMENT_KEY] = core_element
# Set as configured
self.core_element_configured = True
def __configure_core_element__(self, kwargs, user_function):
# type: (dict, ...) -> None
""" Include the registering info related to @binary.
IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY].
:param kwargs: Keyword arguments received from call.
:param user_function: Decorated function.
:return: None
"""
if __debug__:
logger.debug("Configuring @binary core element.")
# Resolve the working directory
self.__resolve_working_dir__()
_working_dir = self.kwargs['working_dir']
# Resolve the fail by exit value
self.__resolve_fail_by_exit_value__()
_fail_by_ev = self.kwargs['fail_by_exit_value']
# Resolve binary
_binary = str(self.kwargs['binary'])
if CORE_ELEMENT_KEY in kwargs and \
kwargs[CORE_ELEMENT_KEY].get_impl_type() == 'CONTAINER':
# @container decorator sits on top of @binary decorator
# Note: impl_type and impl_signature are NOT modified
# ('CONTAINER' and 'CONTAINER.function_name' respectively)
impl_args = kwargs[CORE_ELEMENT_KEY].get_impl_type_args()
_engine = impl_args[0]
_image = impl_args[1]
impl_args = [
_engine, # engine
_image, # image
'CET_BINARY', # internal_type
_binary, # internal_binary
'[unassigned]', # internal_func
_working_dir, # working_dir
_fail_by_ev
] # fail_by_ev
kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args)
else:
# @container decorator does NOT sit on top of @binary decorator
_binary = str(self.kwargs['binary'])
impl_type = 'BINARY'
impl_signature = '.'.join((impl_type, _binary))
impl_args = [
_binary, # internal_binary
_working_dir, # working_dir
_fail_by_ev
] # fail_by_ev
if CORE_ELEMENT_KEY in kwargs:
# Core element has already been created in a higher level
# decorator (e.g. @constraint)
kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type)
kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature)
kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args)
else:
# @binary is in the top of the decorators stack.
# Instantiate a new core element object, update it and include
# it into kwarg
core_element = CE()
core_element.set_impl_type(impl_type)
core_element.set_impl_signature(impl_signature)
core_element.set_impl_type_args(impl_args)
kwargs[CORE_ELEMENT_KEY] = core_element
# Set as configured
self.core_element_configured = True
def __configure_core_element__(self, kwargs, user_function):
# type: (dict, ...) -> None
""" Include the registering info related to @decaf.
IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY].
:param kwargs: Keyword arguments received from call.
:param user_function: Decorated function.
:return: None
"""
if __debug__:
logger.debug("Configuring @decaf core element.")
# Resolve @decaf specific parameters
if 'runner' in self.kwargs:
runner = self.kwargs['runner']
else:
runner = 'mpirun'
if 'dfScript' in self.kwargs:
df_script = self.kwargs['dfScript']
else:
df_script = self.kwargs['df_script']
if 'df_executor' in self.kwargs:
df_executor = self.kwargs['df_executor']
elif 'dfExecutor' in self.kwargs:
df_executor = self.kwargs['dfExecutor']
else:
df_executor = '[unassigned]' # Empty or '[unassigned]'
if 'df_lib' in self.kwargs:
df_lib = self.kwargs['df_lib']
elif 'dfLib' in self.kwargs:
df_lib = self.kwargs['dfLib']
else:
df_lib = '[unassigned]' # Empty or '[unassigned]'
# Resolve the working directory
self.__resolve_working_dir__()
# Resolve the fail by exit value
self.__resolve_fail_by_exit_value__()
impl_type = 'DECAF'
impl_signature = '.'.join((impl_type, df_script))
impl_args = [
df_script, df_executor, df_lib, self.kwargs['working_dir'], runner,
self.kwargs['fail_by_exit_value']
]
if CORE_ELEMENT_KEY in kwargs:
# Core element has already been created in a higher level decorator
# (e.g. @constraint)
kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type)
kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature)
kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args)
else:
# @binary is in the top of the decorators stack.
# Instantiate a new core element object, update it and include
# it into kwarg
core_element = CE()
core_element.set_impl_type(impl_type)
core_element.set_impl_signature(impl_signature)
core_element.set_impl_type_args(impl_args)
kwargs[CORE_ELEMENT_KEY] = core_element
# Set as configured
self.core_element_configured = True
def __configure_core_element__(self, kwargs, user_function):
# type: (dict, ...) -> None
""" Include the registering info related to @compss.
IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY].
:param kwargs: Keyword arguments received from call.
:param user_function: Decorated function.
:return: None
"""
if __debug__:
logger.debug("Configuring @compss core element.")
# Resolve @compss specific parameters
if 'runcompss' in self.kwargs:
runcompss = self.kwargs['runcompss']
else:
runcompss = '[unassigned]' # Empty or '[unassigned]'
if 'flags' in self.kwargs:
flags = self.kwargs['flags']
else:
flags = '[unassigned]' # Empty or '[unassigned]'
if 'worker_in_master' in self.kwargs:
worker_in_master = self.kwargs['worker_in_master']
elif 'workerInMaster' in self.kwargs:
worker_in_master = self.kwargs['workerInMaster']
else:
worker_in_master = 'true' # Empty or '[unassigned]'
if 'appName' in self.kwargs:
app_name = self.kwargs['appName']
else:
app_name = self.kwargs['app_name']
# Resolve the working directory
self.__resolve_working_dir__()
# Resolve the fail by exit value
self.__resolve_fail_by_exit_value__()
impl_type = 'COMPSs'
impl_signature = '.'.join((impl_type, app_name))
impl_args = [runcompss,
flags,
app_name,
worker_in_master,
self.kwargs['working_dir'],
self.kwargs['fail_by_exit_value']]
if CORE_ELEMENT_KEY in kwargs:
# Core element has already been created in a higher level decorator
# (e.g. @constraint)
kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type)
kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature)
kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args)
else:
# @binary is in the top of the decorators stack.
# Instantiate a new core element object, update it and include
# it into kwarg
core_element = CE()
core_element.set_impl_type(impl_type)
core_element.set_impl_signature(impl_signature)
core_element.set_impl_type_args(impl_args)
kwargs[CORE_ELEMENT_KEY] = core_element
# Set as configured
self.core_element_configured = True
def __configure_core_element__(self, kwargs):
# type: (dict) -> None
""" Include the registering info related to @binary.
IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY].
:param kwargs: Keyword arguments received from call.
:return: None
"""
if __debug__:
logger.debug("Configuring @binary core element.")
# Resolve the working directory
resolve_working_dir(self.kwargs)
_working_dir = self.kwargs[WORKING_DIR]
# Resolve the fail by exit value
resolve_fail_by_exit_value(self.kwargs)
_fail_by_ev = self.kwargs[FAIL_BY_EXIT_VALUE]
# Resolve binary
_binary = str(self.kwargs[BINARY])
if CORE_ELEMENT_KEY in kwargs and \
kwargs[CORE_ELEMENT_KEY].get_impl_type() == IMPL_CONTAINER:
# @container decorator sits on top of @binary decorator
# Note: impl_type and impl_signature are NOT modified
# (IMPL_CONTAINER and "CONTAINER.function_name" respectively)
impl_args = kwargs[CORE_ELEMENT_KEY].get_impl_type_args()
_engine = impl_args[0]
_image = impl_args[1]
impl_args = [
_engine, # engine
_image, # image
IMPL_CET_BINARY, # internal_type
_binary, # internal_binary
UNASSIGNED, # internal_func
_working_dir, # working_dir
_fail_by_ev
] # fail_by_ev
kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args)
else:
# @container decorator does NOT sit on top of @binary decorator
_binary = str(self.kwargs[BINARY])
impl_type = IMPL_BINARY
impl_signature = ".".join((impl_type, _binary))
impl_args = [
_binary, # internal_binary
_working_dir, # working_dir
self.kwargs.get('params', UNASSIGNED), # params
_fail_by_ev
] # fail_by_ev
if CORE_ELEMENT_KEY in kwargs:
# Core element has already been created in a higher level
# decorator (e.g. @constraint)
kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type)
kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature)
kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args)
else:
# @binary is in the top of the decorators stack.
# Instantiate a new core element object, update it and include
# it into kwarg
core_element = CE()
core_element.set_impl_type(impl_type)
core_element.set_impl_signature(impl_signature)
core_element.set_impl_type_args(impl_args)
kwargs[CORE_ELEMENT_KEY] = core_element
# Set as configured
self.core_element_configured = True
def __configure_core_element__(self, kwargs):
# type: (dict) -> None
""" Include the registering info related to @compss.
IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY].
:param kwargs: Keyword arguments received from call.
:return: None
"""
if __debug__:
logger.debug("Configuring @compss core element.")
# Resolve @compss specific parameters
if RUNCOMPSS in self.kwargs:
runcompss = self.kwargs[RUNCOMPSS]
else:
runcompss = UNASSIGNED # Empty or UNASSIGNED
if FLAGS in self.kwargs:
flags = self.kwargs[FLAGS]
else:
flags = UNASSIGNED # Empty or UNASSIGNED
if WORKER_IN_MASTER in self.kwargs:
worker_in_master = self.kwargs[WORKER_IN_MASTER]
elif LEGACY_WORKER_IN_MASTER in self.kwargs:
worker_in_master = self.kwargs[LEGACY_WORKER_IN_MASTER]
else:
worker_in_master = "true" # Empty or UNASSIGNED
if LEGACY_APP_NAME in self.kwargs:
app_name = self.kwargs[LEGACY_APP_NAME]
else:
app_name = self.kwargs[APP_NAME]
# Resolve the working directory
resolve_working_dir(self.kwargs)
# Resolve the fail by exit value
resolve_fail_by_exit_value(self.kwargs)
impl_type = IMPL_COMPSs
impl_signature = ".".join((impl_type, app_name))
impl_args = [
runcompss, flags, app_name, worker_in_master,
self.kwargs[WORKING_DIR], self.kwargs[FAIL_BY_EXIT_VALUE]
]
if CORE_ELEMENT_KEY in kwargs:
# Core element has already been created in a higher level decorator
# (e.g. @constraint)
kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type)
kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature)
kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args)
else:
# @binary is in the top of the decorators stack.
# Instantiate a new core element object, update it and include
# it into kwarg
core_element = CE()
core_element.set_impl_type(impl_type)
core_element.set_impl_signature(impl_signature)
core_element.set_impl_type_args(impl_args)
kwargs[CORE_ELEMENT_KEY] = core_element
# Set as configured
self.core_element_configured = True
def __configure_core_element__(self, kwargs, user_function):
# type: (dict, ...) -> None
""" Include the registering info related to @mpi.
IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY].
:param kwargs: Keyword arguments received from call.
:param user_function: Decorated function.
:return: None
"""
if __debug__:
logger.debug("Configuring @mpi core element.")
# Resolve @mpi specific parameters
if "binary" in self.kwargs:
binary = self.kwargs['binary']
impl_type = "MPI"
else:
binary = "[unassigned]"
impl_type = "PYTHON_MPI"
self.task_type = impl_type
runner = self.kwargs['runner']
if 'flags' in self.kwargs:
flags = self.kwargs['flags']
else:
flags = '[unassigned]' # Empty or '[unassigned]'
# Check if scale by cu is defined
scale_by_cu_str = self.__resolve_scale_by_cu__()
# Resolve the working directory
self.__resolve_working_dir__()
# Resolve the fail by exit value
self.__resolve_fail_by_exit_value__()
# Resolve parameter collection layout
collection_layout_params = self.__resolve_collection_layout_params__()
if "processes" in self.kwargs:
proc = self.kwargs["processes"]
elif "computing_nodes" in self.kwargs:
proc = self.kwargs["computing_nodes"]
elif "computingNodes" in self.kwargs:
proc = self.kwargs["computingNodes"]
else:
proc = "1"
if binary == "[unassigned]":
impl_signature = impl_type + '.'
else:
impl_signature = '.'.join((impl_type, str(proc), binary))
impl_args = [
binary, self.kwargs['working_dir'], runner, flags, scale_by_cu_str,
self.kwargs['fail_by_exit_value']
]
if impl_type == "PYTHON_MPI":
impl_args = impl_args + collection_layout_params
if CORE_ELEMENT_KEY in kwargs:
# Core element has already been created in a higher level decorator
# (e.g. @constraint)
kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type)
kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature)
kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args)
else:
# @binary is in the top of the decorators stack.
# Instantiate a new core element object, update it and include
# it into kwarg
core_element = CE()
core_element.set_impl_type(impl_type)
core_element.set_impl_signature(impl_signature)
core_element.set_impl_type_args(impl_args)
kwargs[CORE_ELEMENT_KEY] = core_element
# Set as configured
self.core_element_configured = True
def __init__(self, *args, **kwargs): # noqa
# type: (*typing.Any, **typing.Any) -> None
""" Task constructor.
This part is called in the decoration process, not as an
explicit function call.
We do two things here:
a) Assign default values to unspecified fields.
b) Transform the parameters from user friendly types
(i.e Parameter.IN, etc) to a more convenient internal
representation.
:param args: Decorator positional parameters (ignored).
:param kwargs: Decorator parameters. A task decorator has no positional
arguments.
"""
self.task_type = IMPL_METHOD
self.decorator_name = "".join(("@", Task.__name__.lower()))
self.args = args
self.kwargs = kwargs
self.scope = context.in_pycompss()
self.core_element = CE()
self.core_element_configured = False
# Set missing values to their default ones (step a)
self.decorator_arguments = kwargs
default_decorator_values = {
"target_direction": parameter.INOUT,
"returns": False,
"cache_returns": True,
"priority": False,
"defaults": {},
"time_out": 0,
"is_replicated": False,
"is_distributed": False,
"computing_nodes": 1,
"is_reduce": False,
"chunk_size": 0,
"tracing_hook": False,
"numba": False, # numba mode (jit, vectorize, guvectorize)
"numba_flags": {}, # user defined extra numba flags
"numba_signature": None, # vectorize and guvectorize signature
"numba_declaration": None, # guvectorize declaration
"varargs_type": parameter.IN, # Here for legacy purposes
} # type: typing.Dict[str, typing.Any]
for (key, value) in default_decorator_values.items():
if key not in self.decorator_arguments:
self.decorator_arguments[key] = value
# Give all parameters a unique instance for them (step b)
# Note that when a user defines a task like
# @task(a = IN, b = IN, c = INOUT)
# both a and b point to the same IN object (the one from parameter.py)
# Giving them a unique instance makes life easier in further steps
for (key, value) in self.decorator_arguments.items():
# Not all decorator arguments are necessarily parameters
# (see default_decorator_values)
if is_param(value):
self.decorator_arguments[key] = get_new_parameter(value.key)
# Specific case when value is a dictionary
# Use case example:
# @binary(binary="ls")
# @task(hide={Type: FILE_IN, Prefix: "--hide="},
# sort={Type: IN, Prefix: "--sort="})
# def myLs(flag, hide, sort):
# pass
# Transform this dictionary to a Parameter object
if isinstance(value, dict):
if key not in [
"numba", "numba_flags", "numba_signature",
"numba_declaration"
]:
# Perform user -> instance substitution
# param = self.decorator_arguments[key][parameter.Type]
# Replace the whole dict by a single parameter object
self.decorator_arguments[key] = \
get_parameter_from_dictionary(
self.decorator_arguments[key]
)
else:
# It is a reserved word that we need to keep the user
# defined value (not a Parameter object)
self.decorator_arguments[key] = value
self.user_function = dummy_function # type: typing.Callable
# Global variables common for all tasks of this kind
self.registered = False
self.signature = ""
# Saved from the initial task
self.interactive = False
self.module = None # type: typing.Any
self.function_arguments = tuple() # type: tuple
self.function_name = ""
self.module_name = ""
self.function_type = -1
self.class_name = ""
self.hints = tuple() # type: tuple
self.on_failure = ""
self.defaults = dict() # type: dict
def __configure_core_element__(self, kwargs):
# type: (dict) -> None
""" Include the registering info related to @decaf.
IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY].
:param kwargs: Keyword arguments received from call.
:return: None
"""
if __debug__:
logger.debug("Configuring @decaf core element.")
# Resolve @decaf specific parameters
if RUNNER in self.kwargs:
runner = self.kwargs[RUNNER]
else:
runner = "mpirun"
if LEGACY_DF_SCRIPT in self.kwargs:
df_script = self.kwargs[LEGACY_DF_SCRIPT]
else:
df_script = self.kwargs[DF_SCRIPT]
if DF_EXECUTOR in self.kwargs:
df_executor = self.kwargs[DF_EXECUTOR]
elif LEGACY_DF_EXECUTOR in self.kwargs:
df_executor = self.kwargs[LEGACY_DF_EXECUTOR]
else:
df_executor = UNASSIGNED # Empty or UNASSIGNED
if DF_LIB in self.kwargs:
df_lib = self.kwargs[DF_LIB]
elif LEGACY_DF_LIB in self.kwargs:
df_lib = self.kwargs[LEGACY_DF_LIB]
else:
df_lib = UNASSIGNED # Empty or UNASSIGNED
# Resolve the working directory
resolve_working_dir(self.kwargs)
# Resolve the fail by exit value
resolve_fail_by_exit_value(self.kwargs)
impl_type = IMPL_DECAF
impl_signature = ".".join((impl_type, df_script))
impl_args = [
df_script, df_executor, df_lib, self.kwargs[WORKING_DIR], runner,
self.kwargs[FAIL_BY_EXIT_VALUE]
]
if CORE_ELEMENT_KEY in kwargs:
# Core element has already been created in a higher level decorator
# (e.g. @constraint)
kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type)
kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature)
kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args)
else:
# @binary is in the top of the decorators stack.
# Instantiate a new core element object, update it and include
# it into kwarg
core_element = CE()
core_element.set_impl_type(impl_type)
core_element.set_impl_signature(impl_signature)
core_element.set_impl_type_args(impl_args)
kwargs[CORE_ELEMENT_KEY] = core_element
# Set as configured
self.core_element_configured = True
def __configure_core_element__(self, kwargs):
# type: (dict) -> None
""" Include the registering info related to @mpi.
IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY].
:param kwargs: Keyword arguments received from call.
:return: None
"""
if __debug__:
logger.debug("Configuring @mpi core element.")
# Resolve @mpi specific parameters
if "binary" in self.kwargs:
binary = self.kwargs['binary']
impl_type = "MPI"
else:
binary = "[unassigned]"
impl_type = "PYTHON_MPI"
self.task_type = impl_type
runner = self.kwargs['runner']
if 'flags' in self.kwargs:
flags = self.kwargs['flags']
else:
flags = '[unassigned]' # Empty or '[unassigned]'
if 'scale_by_cu' in self.kwargs:
scale_by_cu = self.kwargs['scale_by_cu']
if isinstance(scale_by_cu, bool):
if scale_by_cu:
scale_by_cu_str = 'true'
else:
scale_by_cu_str = 'false'
elif isinstance(scale_by_cu, str):
scale_by_cu_str = scale_by_cu
else:
raise Exception(
"Incorrect format for scale_by_cu property. "
"It should be boolean or an environment variable"
) # noqa: E501
else:
scale_by_cu_str = 'false'
# Resolve the working directory
self.__resolve_working_dir__()
# Resolve the fail by exit value
self.__resolve_fail_by_exit_value__()
# Resolve parameter collection layout
collection_layout_params = self.__resolve_collection_layout_params__()
if binary == "[unassigned]":
impl_signature = impl_type + '.'
else:
impl_signature = '.'.join(
(impl_type, str(self.kwargs['processes']), binary))
impl_args = [
binary, self.kwargs['working_dir'], runner, flags, scale_by_cu_str,
self.kwargs['fail_by_exit_value']
]
if impl_type == "PYTHON_MPI":
impl_args = impl_args + collection_layout_params
if CORE_ELEMENT_KEY in kwargs:
# Core element has already been created in a higher level decorator
# (e.g. @constraint)
kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type)
kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature)
kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args)
else:
# @binary is in the top of the decorators stack.
# Instantiate a new core element object, update it and include
# it into kwarg
core_element = CE()
core_element.set_impl_type(impl_type)
core_element.set_impl_signature(impl_signature)
core_element.set_impl_type_args(impl_args)
kwargs[CORE_ELEMENT_KEY] = core_element
# Set as configured
self.core_element_configured = True
def test_core_element():
signature = "my_signature"
impl_signature = "my_impl_signature"
impl_constraints = "impl_constraints"
impl_type = "impl_type"
impl_io = "impl_io"
impl_type_args = "impl_type_args"
core_element = CE(signature,
impl_signature,
impl_constraints,
impl_type,
impl_io,
impl_type_args)
# Check signature
result = core_element.get_ce_signature()
assert result == signature, ERROR_SIGNATURE
new_signature = "my_new_signature"
core_element.set_ce_signature(new_signature)
result = core_element.get_ce_signature()
assert result == new_signature, ERROR_SIGNATURE
# Check impl_signature
result = core_element.get_impl_signature()
assert result == impl_signature, ERROR_IMPL_SIGNATURE
new_impl_signature = "my_new_impl_signature"
core_element.set_impl_signature(new_impl_signature)
result = core_element.get_impl_signature()
assert result == new_impl_signature, ERROR_IMPL_SIGNATURE
# Check impl_constraints
result = core_element.get_impl_constraints()
assert result == impl_constraints, ERROR_IMPL_CONSTRAINTS
new_impl_constraints = {"my_new_impl_constraints": "value"}
core_element.set_impl_constraints(new_impl_constraints)
result = core_element.get_impl_constraints()
assert result == new_impl_constraints, ERROR_IMPL_CONSTRAINTS
# Check impl_type
result = core_element.get_impl_type()
assert result == impl_type, ERROR_IMPL_TYPE
new_impl_type = "my_new_impl_type"
core_element.set_impl_type(new_impl_type)
result = core_element.get_impl_type()
assert result == new_impl_type, ERROR_IMPL_TYPE
# Check impl_io
result = core_element.get_impl_io()
assert result == impl_io, ERROR_IMPL_IO
new_impl_io = "my_new_impl_io"
core_element.set_impl_io(new_impl_io)
result = core_element.get_impl_io()
assert result == new_impl_io, ERROR_IMPL_IO
# Check impl_type_args
result = core_element.get_impl_type_args()
assert result == impl_type_args, ERROR_IMPL_TYPE_ARGS
new_impl_type_args = "my_new_impl_type_args"
core_element.set_impl_type_args(new_impl_type_args)
result = core_element.get_impl_type_args()
assert result == new_impl_type_args, ERROR_IMPL_TYPE_ARGS
# Check representation
representation = core_element.__repr__()
assert isinstance(representation, str), "ERROR: Received wrong representation type." # noqa: E501
expected = "CORE ELEMENT: \n" \
"\t - CE signature : my_new_signature\n" \
"\t - Impl. signature : my_new_impl_signature\n" \
"\t - Impl. constraints: my_new_impl_constraints:value;\n" \
"\t - Impl. type : my_new_impl_type\n" \
"\t - Impl. io : my_new_impl_io\n" \
"\t - Impl. type args : my_new_impl_type_args"
assert representation == expected, "ERROR: Wrong representation."
# Reset
core_element.reset()
# Check again representation
representation = core_element.__repr__()
assert isinstance(representation, str), "ERROR: Received wrong representation type." # noqa: E501
expected = "CORE ELEMENT: \n" \
"\t - CE signature : None\n" \
"\t - Impl. signature : None\n" \
"\t - Impl. constraints: None\n" \
"\t - Impl. type : None\n" \
"\t - Impl. io : None\n" \
"\t - Impl. type args : None"
assert representation == expected, "ERROR: Wrong empty representation."
def __configure_core_element__(self, kwargs):
# type: (dict) -> None
""" Include the registering info related to @mpi.
IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY].
:param kwargs: Keyword arguments received from call.
:return: None
"""
if __debug__:
logger.debug("Configuring @mpi core element.")
# Resolve @mpi specific parameters
if BINARY in self.kwargs:
binary = self.kwargs[BINARY]
impl_type = IMPL_MPI
else:
binary = UNASSIGNED
impl_type = IMPL_PYTHON_MPI
self.task_type = impl_type
runner = self.kwargs[RUNNER]
if FLAGS in self.kwargs:
flags = self.kwargs[FLAGS]
else:
flags = UNASSIGNED # Empty or UNASSIGNED
# Check if scale by cu is defined
scale_by_cu_str = self.__resolve_scale_by_cu__()
# Resolve the working directory
resolve_working_dir(self.kwargs)
# Resolve the fail by exit value
resolve_fail_by_exit_value(self.kwargs)
# Resolve parameter collection layout
collection_layout_params = self.__resolve_collection_layout_params__()
if "processes" in self.kwargs:
proc = self.kwargs["processes"]
elif "computing_nodes" in self.kwargs:
proc = self.kwargs["computing_nodes"]
elif "computingNodes" in self.kwargs:
proc = self.kwargs["computingNodes"]
else:
proc = "1"
if "processes_per_node" in self.kwargs:
ppn = str(self.kwargs["processes_per_node"])
else:
ppn = "1"
if binary == UNASSIGNED:
impl_signature = impl_type + '.'
else:
impl_signature = '.'.join((impl_type, str(proc), binary))
impl_args = [
binary, self.kwargs[WORKING_DIR], runner, ppn, flags,
scale_by_cu_str,
self.kwargs.get("params",
UNASSIGNED), self.kwargs[FAIL_BY_EXIT_VALUE]
]
if impl_type == IMPL_PYTHON_MPI:
impl_args = impl_args + collection_layout_params
if CORE_ELEMENT_KEY in kwargs:
# Core element has already been created in a higher level decorator
# (e.g. @constraint)
kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type)
kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature)
kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args)
else:
# @binary is in the top of the decorators stack.
# Instantiate a new core element object, update it and include
# it into kwarg
core_element = CE()
core_element.set_impl_type(impl_type)
core_element.set_impl_signature(impl_signature)
core_element.set_impl_type_args(impl_args)
kwargs[CORE_ELEMENT_KEY] = core_element
# Set as configured
self.core_element_configured = True