Esempio n. 1
0
def c_extension_link(in_queue, out_queue, redirect_std, out_file_name,
                     err_file_name):
    # type: (Queue, Queue, bool, str, str) -> None
    """ Main C extension process.

    :param in_queue: Queue to receive messages.
    :param out_queue: Queue to send messages.
    :param redirect_std: Decide whether to store the stdout and stderr into
                         files or not.
    :param out_file_name: File where to store the stdout (only required if
                          redirect_std is True).
    :param err_file_name: File where to store the stderr (only required if
                          redirect_std is True).
    :return: None
    """
    # Import C extension within the external process
    import compss

    with ipython_std_redirector(out_file_name, err_file_name) \
            if redirect_std else not_std_redirector():
        alive = True
        while alive:
            message = in_queue.get()
            command = message[0]
            parameters = []  # type: list
            if len(message) > 0:
                parameters = list(message[1:])
            if command == START:
                compss.start_runtime()
            elif command == SET_DEBUG:
                compss.set_debug(*parameters)
            elif command == STOP:
                compss.stop_runtime(*parameters)
                alive = False
            elif command == CANCEL_TASKS:
                compss.cancel_application_tasks(*parameters)
            elif command == ACCESSED_FILE:
                accessed = compss.accessed_file(*parameters)
                out_queue.put(accessed)
            elif command == OPEN_FILE:
                compss_name = compss.open_file(*parameters)
                out_queue.put(compss_name)
            elif command == CLOSE_FILE:
                compss.close_file(*parameters)
            elif command == DELETE_FILE:
                result = compss.delete_file(*parameters)
                out_queue.put(result)
            elif command == GET_FILE:
                compss.get_file(*parameters)
            elif command == GET_DIRECTORY:
                compss.get_directory(*parameters)
            elif command == BARRIER:
                compss.barrier(*parameters)
            elif command == BARRIER_GROUP:
                exception_message = compss.barrier_group(*parameters)
                out_queue.put(exception_message)
            elif command == OPEN_TASK_GROUP:
                compss.open_task_group(*parameters)
            elif command == CLOSE_TASK_GROUP:
                compss.close_task_group(*parameters)
            elif command == GET_LOGGING_PATH:
                log_path = compss.get_logging_path()
                out_queue.put(log_path)
            elif command == GET_NUMBER_OF_RESOURCES:
                num_resources = compss.get_number_of_resources(*parameters)
                out_queue.put(num_resources)
            elif command == REQUEST_RESOURCES:
                compss.request_resources(*parameters)
            elif command == FREE_RESOURCES:
                compss.free_resources(*parameters)
            elif command == REGISTER_CORE_ELEMENT:
                compss.register_core_element(*parameters)
            elif command == PROCESS_TASK:
                compss.process_task(*parameters)
            elif command == PROCESS_HTTP_TASK:
                compss.process_http_task(*parameters)
            elif command == SET_PIPES:
                compss.set_pipes(*parameters)
            elif command == READ_PIPES:
                compss.read_pipes(*parameters)
            elif command == SET_WALL_CLOCK:
                compss.set_wall_clock(*parameters)
            else:
                raise PyCOMPSsException("Unknown link command")
Esempio n. 2
0
def process_task(f, module_name, class_name, ftype, f_parameters, f_returns,
                 task_kwargs, num_nodes, replicated, distributed):
    """
    Function that submits a task to the runtime.

    :param f: Function or method
    :param module_name: Name of the module containing the function/method (including packages, if any)
    :param class_name: Name of the class (if method)
    :param ftype: Function type
    :param f_parameters: Function parameters (dictionary {'param1':Parameter()}
    :param f_returns: Function returns (dictionary {'compss_retvalueX':Parameter()}
    :param task_kwargs: Decorator arguments
    :param num_nodes: Number of nodes that the task must use
    :param replicated: Boolean indicating if the task must be replicated or not
    :param distributed: Boolean indicating if the task must be distributed or not
    :return: The future object related to the task return
    """

    if __debug__:
        logger.debug("TASK: %s of type %s, in module %s, in class %s" %
                     (f.__name__, ftype, module_name, class_name))

    app_id = 0

    # Check if the function is an instance method or a class method.
    if ftype == FunctionType.INSTANCE_METHOD:
        has_target = True
    else:
        has_target = False

    fo = None
    if f_returns:
        fo = _build_return_objects(f_returns)

    num_returns = len(f_returns)

    # Get path
    if class_name == '':
        path = module_name
    else:
        path = module_name + '.' + class_name

    # Infer COMPSs types from real types, except for files
    _serialize_objects(f_parameters)

    # Build values and COMPSs types and directions
    values, compss_types, compss_directions, compss_streams, compss_prefixes = _build_values_types_directions(
        ftype, f_parameters, f_returns, f.__code_strings__)

    # Get priority
    has_priority = task_kwargs['priority']

    # Signature and other parameters:
    signature = '.'.join([path, f.__name__])
    # num_nodes = 1        # default due to not MPI decorator yet
    # replicated = False   # default due to not replicated tag yet
    # distributed = False  # default due to not distributed tag yet

    if __debug__:
        # Log the task submission values for debugging purposes.
        if logger.isEnabledFor(logging.DEBUG):
            values_str = ''
            types_str = ''
            direct_str = ''
            streams_str = ''
            prefixes_str = ''
            for v in values:
                values_str += str(v) + ' '
            for t in compss_types:
                types_str += str(t) + ' '
            for d in compss_directions:
                direct_str += str(d) + ' '
            for s in compss_streams:
                streams_str += str(s) + ' '
            for p in compss_prefixes:
                prefixes_str += str(p) + ' '
            logger.debug("Processing task:")
            logger.debug("\t- App id: " + str(app_id))
            logger.debug("\t- Path: " + path)
            logger.debug("\t- Function name: " + f.__name__)
            logger.debug("\t- Signature: " + signature)
            logger.debug("\t- Priority: " + str(has_priority))
            logger.debug("\t- Has target: " + str(has_target))
            logger.debug("\t- Num nodes: " + str(num_nodes))
            logger.debug("\t- Replicated: " + str(replicated))
            logger.debug("\t- Distributed: " + str(distributed))
            logger.debug("\t- Values: " + values_str)
            logger.debug("\t- COMPSs types: " + types_str)
            logger.debug("\t- COMPSs directions: " + direct_str)
            logger.debug("\t- COMPSs streams: " + streams_str)
            logger.debug("\t- COMPSs prefixes: " + prefixes_str)

    # Check that there is the same amount of values as their types, as well as their directions, streams and prefixes.
    assert (len(values) == len(compss_types) == len(compss_directions) ==
            len(compss_streams) == len(compss_prefixes))
    '''
    Submit task to the runtime (call to the C extension):
    Parameters:
        0 - <Integer>   - application id (by default always 0 due to it is not currently needed for the signature)
        1 - <String>    - path of the module where the task is
        2 - <String>    - function name of the task (to be called from the worker)
        3 - <String>    - priority flag (true|false)
        4 - <String>    - has target (true|false). If the task is within an object or not.
        5 - [<String>]  - task parameters (basic types or file paths for objects)
        6 - [<Integer>] - parameters types (number corresponding to the type of each parameter)
        7 - [<Integer>] - parameters directions (number corresponding to the direction of each parameter)
        8 - [<Integer>] - parameters streams (number corresponding to the stream of each parameter)
        9 - [<String>]  - parameters prefixes (sting corresponding to the prefix of each parameter)
    '''

    compss.process_task(app_id, signature, has_priority, num_nodes, replicated,
                        distributed, has_target, num_returns, values,
                        compss_types, compss_directions, compss_streams,
                        compss_prefixes)

    # Return the future object/s corresponding to the task
    # This object will substitute the user expected return from the task and will be used later for synchronization
    # or as a task parameter (then the runtime will take care of the dependency.
    return fo
Esempio n. 3
0
def c_extension_link(in_queue, out_queue):
    # type: (..., ...) -> None
    """ Main C extension process.

    :param in_queue: Queue to receive messages.
    :param out_queue: Queue to send messages.
    :return: None
    """
    import compss

    alive = True
    while alive:
        message = in_queue.get()
        command = message[0]
        parameters = []
        if len(message) > 0:
            parameters = message[1:]
        if command == START:
            compss.start_runtime()
        elif command == SET_DEBUG:
            compss.set_debug(*parameters)
        elif command == STOP:
            compss.stop_runtime(*parameters)
            alive = False
        elif command == CANCEL_TASKS:
            compss.cancel_application_tasks(*parameters)
        elif command == ACCESSED_FILE:
            accessed = compss.accessed_file(*parameters)
            out_queue.put(accessed)
        elif command == OPEN_FILE:
            compss_name = compss.open_file(*parameters)
            out_queue.put(compss_name)
        elif command == CLOSE_FILE:
            compss.close_file(*parameters)
        elif command == DELETE_FILE:
            result = compss.delete_file(*parameters)
            out_queue.put(result)
        elif command == GET_FILE:
            compss.get_file(*parameters)
        elif command == GET_DIRECTORY:
            compss.get_directory(*parameters)
        elif command == BARRIER:
            compss.barrier(*parameters)
        elif command == BARRIER_GROUP:
            exception_message = compss.barrier_group(*parameters)
            out_queue.put(exception_message)
        elif command == OPEN_TASK_GROUP:
            compss.open_task_group(*parameters)
        elif command == CLOSE_TASK_GROUP:
            compss.close_task_group(*parameters)
        elif command == GET_LOGGING_PATH:
            log_path = compss.get_logging_path()
            out_queue.put(log_path)
        elif command == GET_NUMBER_OF_RESOURCES:
            num_resources = compss.get_number_of_resources(*parameters)
            out_queue.put(num_resources)
        elif command == REQUEST_RESOURCES:
            compss.request_resources(*parameters)
        elif command == FREE_RESOURCES:
            compss.free_resources(*parameters)
        elif command == REGISTER_CORE_ELEMENT:
            compss.register_core_element(*parameters)
        elif command == PROCESS_TASK:
            compss.process_task(*parameters)
        elif command == SET_PIPES:
            compss.set_pipes(*parameters)
        else:
            raise Exception("Unknown link command")
def process_task(f, # Function or method
                 ftype, # Function type
                 spec_args, # Names of the task arguments
                 class_name, # Name of the class (if method)
                 module_name, # Name of the module containing the function/method (including packages, if any)
                 task_args, # Unnamed arguments
                 task_kwargs, # Named arguments
                 deco_kwargs): # Decorator arguments
    
    logger.debug("TASK: %s of type %s, in module %s, in class %s" % (f.__name__, ftype, module_name, class_name))
    
    first_par = 0
    if ftype == Function_Type.INSTANCE_METHOD:
        has_target = True
    else:
        has_target = False
        if ftype == Function_Type.CLASS_METHOD:
            first_par = 1 # skip class parameter
    
    ret_type = deco_kwargs['returns'] 
    if ret_type:
        # Create future for return value
        if ret_type in python_to_compss: # primitives, string, dic, list, tuple
            fu = Future()
        elif inspect.isclass(ret_type):
            # For objects, type of future has to be specified to allow o = func; o.func
            try:
                fu = ret_type()
            except TypeError:
                logger.warning("Type %s does not have an empty constructor, building generic future object" % ret_type)
                fu = Future()
        else:
            fu = Future() # modules, functions, methods
        
        logger.debug("Setting object %d of %s as a future" % (id(fu), type(fu)))
        
        obj_id = id(fu)
        ret_filename = temp_dir + temp_obj_prefix + str(obj_id)
        objid_to_filename[obj_id] = ret_filename
        task_objects[obj_id] = fu
        task_kwargs['compss_retvalue'] = ret_filename
    else:
        fu = None
    
    app_id = 0
 
    if class_name == '':
	path = module_name
    else:
        path = module_name + '.' + class_name
    
    # Infer COMPSs types from real types, except for files
    num_pars = len(spec_args)
    is_future = {}
    for i in range(first_par, num_pars):
        spec_arg = spec_args[i]
        p = deco_kwargs.get(spec_arg)
        if p == None:
            logger.debug("Adding default decoration for parameter %s" % spec_arg)
            p = Parameter()
            deco_kwargs[spec_arg] = p
        if i < len(task_args):
            p.value = task_args[i]
        else:
            p.value = task_kwargs[spec_arg]
        
        val_type = type(p.value)
        is_future[i] = (val_type == Future)
        logger.debug("Parameter " + spec_arg + "\n" + 
                     "\t- Value type: " + str(val_type) + "\n" +
                     "\t- User-defined type: " + str(p.type))

        # Infer type if necessary
        if p.type == None:
            p.type = python_to_compss.get(val_type)
            if p.type == None:
                p.type = Type.OBJECT
            logger.debug("\n\t- Inferred type: %d" % p.type)
        
        # Serialize objects into files
        if p.type == Type.OBJECT or is_future.get(i): # 2nd condition: real type can be primitive, but now it's acting as a future (object)
            turn_into_file(p)
	elif p.type == Type.INT:
	    if p.value > JAVA_MAX_INT or p.value < JAVA_MIN_INT:
		p.type = Type.LONG  # this must go through Java as a long to prevent overflow with Java int
	elif p.type == Type.LONG:
	    if p.value > JAVA_MAX_LONG or p.value < JAVA_MIN_LONG:
		p.type = Type.OBJECT # this must be serialized to prevent overflow with Java long
		turn_into_file(p)
        
        logger.debug("Final type for parameter %s: %d" % (spec_arg, p.type))
    
    # Build values and COMPSs types and directions
    values = []
    compss_types = []
    compss_directions = []
    if ftype == Function_Type.INSTANCE_METHOD:
        ra = range(1, num_pars)
        ra.append(0) # callee is the last
    else:
        ra = range(first_par, num_pars)
    for i in ra:
        spec_arg = spec_args[i]
        p = deco_kwargs[spec_arg]
        values.append(p.value)
        if p.type == Type.OBJECT or is_future.get(i):
            compss_types.append(Type.FILE)
        else:
            compss_types.append(p.type)
        compss_directions.append(p.direction)
    
    # Priority
    has_priority = deco_kwargs['priority']
    
    if logger.isEnabledFor(logging.DEBUG):
        values_str = ''
        types_str = ''
        direct_str = ''
        for v in values:
            values_str += str(v) + " "
        for t in compss_types:
            types_str += str(t) + " "
        for d in compss_directions:
            direct_str += str(d) + " "
        logger.debug("Processing task:\n" +
                     "\t- App id: " + str(app_id) + "\n" +
                     "\t- Path: " + path + "\n" +
                     "\t- Function name: " + f.__name__ + "\n" +
                     "\t- Priority: " + str(has_priority) + "\n" +
                     "\t- Has target: " + str(has_target) + "\n" +
                     "\t- Num params: " + str(num_pars) + "\n" +
                     "\t- Values: " + values_str + "\n" +
                     "\t- COMPSs types: " + types_str + "\n" +
                     "\t- COMPSs directions: " + direct_str)
    
    compss.process_task(app_id,
                        path,
                        f.__name__,
                        has_priority,
                        has_target,
                        values, compss_types, compss_directions)
    
    return fu