예제 #1
0
def synchronize(obj, mode):
    '''
    Synchronization function.
    This method retrieves the value of a future object.
    Calls the runtime in order to wait for the value and returns it
    when received.
    :param obj: Object to synchronize.
    :param mode: Direction of the object to synchronize.
    :return: The value of the object requested.
    '''
    # TODO - CUANDO SE LLAME A compss.get_file, anadir un booleano diferenciando si es fichero u objeto
    # Objetivo: mejorar el detalle de las trazas. Esto se tiene que implementar
    # primero en el runtime, despues adaptar el api de C, y finalmente anadir
    # el booleano aqui.
    global current_id

    if is_PSCO(obj):
        obj_id = get_ID(obj)
        if obj_id not in pending_to_synchronize:
            return obj
        else:
            # file_path is of the form storage://pscoID or file://sys_path_to_file
            file_path = compss.get_file('storage://' + str(obj_id), mode)
            # TODO: Add switch on protocol
            protocol, file_name = file_path.split('://')
            new_obj = get_by_ID(file_name)
            return new_obj

    obj_id = get_object_id(obj)
    if obj_id not in pending_to_synchronize:
        return obj

    if __debug__:
        logger.debug('Synchronizing object %s with mode %s' % (obj_id, mode))

    file_name = objid_to_filename[obj_id]
    compss_file = compss.get_file(file_name, mode)

    # Runtime can return a path or a PSCOId
    if compss_file.startswith('/'):
        new_obj = deserialize_from_file(compss_file)
        compss.close_file(file_name, mode)
    else:
        new_obj = get_by_ID(compss_file)
    new_obj_id = get_object_id(new_obj, True, True)

    # The main program won't work with the old object anymore, update mapping
    objid_to_filename[new_obj_id] = objid_to_filename[obj_id].replace(obj_id, new_obj_id)
    objs_written_by_mp[new_obj_id] = objid_to_filename[new_obj_id]

    if __debug__:
        logger.debug('Deleting obj %s (new one is %s)' % (str(obj_id), str(new_obj_id)))
    compss.delete_file(objid_to_filename[obj_id])
    objid_to_filename.pop(obj_id)
    pending_to_synchronize.pop(obj_id)

    if __debug__:
        logger.debug('Now object with id %s and %s has mapping %s' % (new_obj_id, type(new_obj), file_name))

    return new_obj
예제 #2
0
파일: link.py 프로젝트: bsc-wdc/compss
def c_extension_link(in_queue, out_queue, redirect_std, out_file_name,
                     err_file_name):
    # type: (Queue, Queue, bool, str, str) -> None
    """ Main C extension process.

    :param in_queue: Queue to receive messages.
    :param out_queue: Queue to send messages.
    :param redirect_std: Decide whether to store the stdout and stderr into
                         files or not.
    :param out_file_name: File where to store the stdout (only required if
                          redirect_std is True).
    :param err_file_name: File where to store the stderr (only required if
                          redirect_std is True).
    :return: None
    """
    # Import C extension within the external process
    import compss

    with ipython_std_redirector(out_file_name, err_file_name) \
            if redirect_std else not_std_redirector():
        alive = True
        while alive:
            message = in_queue.get()
            command = message[0]
            parameters = []  # type: list
            if len(message) > 0:
                parameters = list(message[1:])
            if command == START:
                compss.start_runtime()
            elif command == SET_DEBUG:
                compss.set_debug(*parameters)
            elif command == STOP:
                compss.stop_runtime(*parameters)
                alive = False
            elif command == CANCEL_TASKS:
                compss.cancel_application_tasks(*parameters)
            elif command == ACCESSED_FILE:
                accessed = compss.accessed_file(*parameters)
                out_queue.put(accessed)
            elif command == OPEN_FILE:
                compss_name = compss.open_file(*parameters)
                out_queue.put(compss_name)
            elif command == CLOSE_FILE:
                compss.close_file(*parameters)
            elif command == DELETE_FILE:
                result = compss.delete_file(*parameters)
                out_queue.put(result)
            elif command == GET_FILE:
                compss.get_file(*parameters)
            elif command == GET_DIRECTORY:
                compss.get_directory(*parameters)
            elif command == BARRIER:
                compss.barrier(*parameters)
            elif command == BARRIER_GROUP:
                exception_message = compss.barrier_group(*parameters)
                out_queue.put(exception_message)
            elif command == OPEN_TASK_GROUP:
                compss.open_task_group(*parameters)
            elif command == CLOSE_TASK_GROUP:
                compss.close_task_group(*parameters)
            elif command == GET_LOGGING_PATH:
                log_path = compss.get_logging_path()
                out_queue.put(log_path)
            elif command == GET_NUMBER_OF_RESOURCES:
                num_resources = compss.get_number_of_resources(*parameters)
                out_queue.put(num_resources)
            elif command == REQUEST_RESOURCES:
                compss.request_resources(*parameters)
            elif command == FREE_RESOURCES:
                compss.free_resources(*parameters)
            elif command == REGISTER_CORE_ELEMENT:
                compss.register_core_element(*parameters)
            elif command == PROCESS_TASK:
                compss.process_task(*parameters)
            elif command == PROCESS_HTTP_TASK:
                compss.process_http_task(*parameters)
            elif command == SET_PIPES:
                compss.set_pipes(*parameters)
            elif command == READ_PIPES:
                compss.read_pipes(*parameters)
            elif command == SET_WALL_CLOCK:
                compss.set_wall_clock(*parameters)
            else:
                raise PyCOMPSsException("Unknown link command")
예제 #3
0
def c_extension_link(in_queue, out_queue):
    # type: (..., ...) -> None
    """ Main C extension process.

    :param in_queue: Queue to receive messages.
    :param out_queue: Queue to send messages.
    :return: None
    """
    import compss

    alive = True
    while alive:
        message = in_queue.get()
        command = message[0]
        parameters = []
        if len(message) > 0:
            parameters = message[1:]
        if command == START:
            compss.start_runtime()
        elif command == SET_DEBUG:
            compss.set_debug(*parameters)
        elif command == STOP:
            compss.stop_runtime(*parameters)
            alive = False
        elif command == CANCEL_TASKS:
            compss.cancel_application_tasks(*parameters)
        elif command == ACCESSED_FILE:
            accessed = compss.accessed_file(*parameters)
            out_queue.put(accessed)
        elif command == OPEN_FILE:
            compss_name = compss.open_file(*parameters)
            out_queue.put(compss_name)
        elif command == CLOSE_FILE:
            compss.close_file(*parameters)
        elif command == DELETE_FILE:
            result = compss.delete_file(*parameters)
            out_queue.put(result)
        elif command == GET_FILE:
            compss.get_file(*parameters)
        elif command == GET_DIRECTORY:
            compss.get_directory(*parameters)
        elif command == BARRIER:
            compss.barrier(*parameters)
        elif command == BARRIER_GROUP:
            exception_message = compss.barrier_group(*parameters)
            out_queue.put(exception_message)
        elif command == OPEN_TASK_GROUP:
            compss.open_task_group(*parameters)
        elif command == CLOSE_TASK_GROUP:
            compss.close_task_group(*parameters)
        elif command == GET_LOGGING_PATH:
            log_path = compss.get_logging_path()
            out_queue.put(log_path)
        elif command == GET_NUMBER_OF_RESOURCES:
            num_resources = compss.get_number_of_resources(*parameters)
            out_queue.put(num_resources)
        elif command == REQUEST_RESOURCES:
            compss.request_resources(*parameters)
        elif command == FREE_RESOURCES:
            compss.free_resources(*parameters)
        elif command == REGISTER_CORE_ELEMENT:
            compss.register_core_element(*parameters)
        elif command == PROCESS_TASK:
            compss.process_task(*parameters)
        elif command == SET_PIPES:
            compss.set_pipes(*parameters)
        else:
            raise Exception("Unknown link command")
예제 #4
0
def synchronize(obj, mode):
    """
    Synchronization function.
    This method retrieves the value of a future object.
    Calls the runtime in order to wait for the value and returns it when received.

    :param obj: Object to synchronize.
    :param mode: Direction of the object to synchronize.
    :return: The value of the object requested.
    """

    # TODO: Add a boolean to differentiate between files and object on the compss.get_file call. This change pretends
    # to obtain better traces. Must be implemented first in the Runtime, then in the bindings common C API and
    # finally add the boolean here
    global _current_id

    if is_psco(obj):
        obj_id = get_id(obj)
        if obj_id not in pending_to_synchronize:
            return obj
        else:
            # file_path is of the form storage://pscoId or file://sys_path_to_file
            file_path = compss.get_file("storage://" + str(obj_id), mode)
            # TODO: Add switch on protocol
            protocol, file_name = file_path.split('://')
            new_obj = get_by_id(file_name)
            return new_obj

    obj_id = get_object_id(obj)
    if obj_id not in pending_to_synchronize:
        return obj

    if __debug__:
        logger.debug("Synchronizing object %s with mode %s" % (obj_id, mode))

    file_name = objid_to_filename[obj_id]
    compss_file = compss.get_file(file_name, mode)

    # Runtime can return a path or a PSCOId
    if compss_file.startswith('/'):
        new_obj = deserialize_from_file(compss_file)
        compss.close_file(file_name, mode)
    else:
        new_obj = get_by_id(compss_file)
    new_obj_id = get_object_id(new_obj, True, True)

    # The main program won't work with the old object anymore, update mapping
    objid_to_filename[new_obj_id] = objid_to_filename[obj_id].replace(
        obj_id, new_obj_id)
    _objs_written_by_mp[new_obj_id] = objid_to_filename[new_obj_id]

    if __debug__:
        logger.debug("Deleting obj %s (new one is %s)" %
                     (str(obj_id), str(new_obj_id)))
    compss.delete_file(objid_to_filename[obj_id])
    objid_to_filename.pop(obj_id)
    pending_to_synchronize.pop(obj_id)

    if __debug__:
        logger.debug("Now object with id %s and %s has mapping %s" %
                     (new_obj_id, type(new_obj), file_name))

    return new_obj