Exemplo n.º 1
0
def synchronize(obj, mode):
    '''
    Synchronization function.
    This method retrieves the value of a future object.
    Calls the runtime in order to wait for the value and returns it
    when received.
    :param obj: Object to synchronize.
    :param mode: Direction of the object to synchronize.
    :return: The value of the object requested.
    '''
    # TODO - CUANDO SE LLAME A compss.get_file, anadir un booleano diferenciando si es fichero u objeto
    # Objetivo: mejorar el detalle de las trazas. Esto se tiene que implementar
    # primero en el runtime, despues adaptar el api de C, y finalmente anadir
    # el booleano aqui.
    global current_id

    if is_PSCO(obj):
        obj_id = get_ID(obj)
        if obj_id not in pending_to_synchronize:
            return obj
        else:
            # file_path is of the form storage://pscoID or file://sys_path_to_file
            file_path = compss.get_file('storage://' + str(obj_id), mode)
            # TODO: Add switch on protocol
            protocol, file_name = file_path.split('://')
            new_obj = get_by_ID(file_name)
            return new_obj

    obj_id = get_object_id(obj)
    if obj_id not in pending_to_synchronize:
        return obj

    if __debug__:
        logger.debug('Synchronizing object %s with mode %s' % (obj_id, mode))

    file_name = objid_to_filename[obj_id]
    compss_file = compss.get_file(file_name, mode)

    # Runtime can return a path or a PSCOId
    if compss_file.startswith('/'):
        new_obj = deserialize_from_file(compss_file)
        compss.close_file(file_name, mode)
    else:
        new_obj = get_by_ID(compss_file)
    new_obj_id = get_object_id(new_obj, True, True)

    # The main program won't work with the old object anymore, update mapping
    objid_to_filename[new_obj_id] = objid_to_filename[obj_id].replace(obj_id, new_obj_id)
    objs_written_by_mp[new_obj_id] = objid_to_filename[new_obj_id]

    if __debug__:
        logger.debug('Deleting obj %s (new one is %s)' % (str(obj_id), str(new_obj_id)))
    compss.delete_file(objid_to_filename[obj_id])
    objid_to_filename.pop(obj_id)
    pending_to_synchronize.pop(obj_id)

    if __debug__:
        logger.debug('Now object with id %s and %s has mapping %s' % (new_obj_id, type(new_obj), file_name))

    return new_obj
Exemplo n.º 2
0
def delete_object(obj):
    """
    Removes a used object from the internal structures and calls the
    external python library (that calls the bindings-common)
    in order to request a its corresponding file removal.

    :param obj: Object to remove
    :return: True if success. False otherwise
    """

    obj_id = get_object_id(obj, False, False)
    if obj_id is None:
        return False

    try:
        _id2obj.pop(obj_id)
    except KeyError:
        pass
    try:
        file_name = objid_to_filename[obj_id]
        compss.delete_file(file_name)
    except KeyError:
        pass
    try:
        objid_to_filename.pop(obj_id)
    except KeyError:
        pass
    try:
        pending_to_synchronize.pop(obj_id)
    except KeyError:
        pass
    return True
Exemplo n.º 3
0
def compss_delete_object(obj):
    import compss
    from pycompss.runtime.binding import get_object_id
    from pycompss.runtime.binding import objid_to_filename
    from pycompss.runtime.binding import pending_to_synchronize
    from pycompss.runtime.binding import id2obj
    obj_id = get_object_id(obj, False, False)
    if obj_id is None:
        return False
    try:
        id2obj.pop(obj_id)
    except:
        pass
    try:
        file_name = objid_to_filename[obj_id]
        compss.delete_file(file_name)
    except:
        pass
    try:
        objid_to_filename.pop(obj_id)
    except:
        pass
    try:
        pending_to_synchronize.pop(obj_id)
    except:
        pass
    return True
Exemplo n.º 4
0
def delete_file(file_name):
    '''
    Calls the external python library (that calls the bindings-common)
    in order to request a file removal.
    :param file_name: File name to remove
    :return: True if success. False on the contrary
    '''
    if __debug__:
        logger.debug('Deleting file %s' % file_name)
    compss.delete_file(file_name)
Exemplo n.º 5
0
def clean_objects():
    '''
    Clean the objects stored in the global dictionaries:
        * pending_to_synchronize dict
        * id2obj dict
        * objid_to_filename dict
        * objs_written_by_mp dict
    '''
    for filename in objid_to_filename.values():
        compss.delete_file(filename)
    pending_to_synchronize.clear()
    id2obj.clear()
    objid_to_filename.clear()
    objs_written_by_mp.clear()
Exemplo n.º 6
0
def _clean_objects():
    """
    Clean the objects stored in the global dictionaries:
        * pending_to_synchronize dict
        * _id2obj dict
        * objid_to_filename dict
        * _objs_written_by_mp dict

    :return: None
    """

    for filename in objid_to_filename.values():
        compss.delete_file(filename)
    pending_to_synchronize.clear()
    _id2obj.clear()
    objid_to_filename.clear()
    _objs_written_by_mp.clear()
Exemplo n.º 7
0
def delete_file(file_name):
    """
    Calls the external python library (that calls the bindings-common)
    in order to request a file removal.

    :param file_name: File name to remove
    :return: True if success. False otherwise
    """

    if __debug__:
        logger.debug("Deleting file %s" % file_name)
    result = compss.delete_file(file_name) == 'true'
    if __debug__:
        if result:
            logger.debug("File %s successfully deleted." % file_name)
        else:
            logger.error("Failed to remove file %s." % file_name)
    return result
Exemplo n.º 8
0
def c_extension_link(in_queue, out_queue, redirect_std, out_file_name,
                     err_file_name):
    # type: (Queue, Queue, bool, str, str) -> None
    """ Main C extension process.

    :param in_queue: Queue to receive messages.
    :param out_queue: Queue to send messages.
    :param redirect_std: Decide whether to store the stdout and stderr into
                         files or not.
    :param out_file_name: File where to store the stdout (only required if
                          redirect_std is True).
    :param err_file_name: File where to store the stderr (only required if
                          redirect_std is True).
    :return: None
    """
    # Import C extension within the external process
    import compss

    with ipython_std_redirector(out_file_name, err_file_name) \
            if redirect_std else not_std_redirector():
        alive = True
        while alive:
            message = in_queue.get()
            command = message[0]
            parameters = []  # type: list
            if len(message) > 0:
                parameters = list(message[1:])
            if command == START:
                compss.start_runtime()
            elif command == SET_DEBUG:
                compss.set_debug(*parameters)
            elif command == STOP:
                compss.stop_runtime(*parameters)
                alive = False
            elif command == CANCEL_TASKS:
                compss.cancel_application_tasks(*parameters)
            elif command == ACCESSED_FILE:
                accessed = compss.accessed_file(*parameters)
                out_queue.put(accessed)
            elif command == OPEN_FILE:
                compss_name = compss.open_file(*parameters)
                out_queue.put(compss_name)
            elif command == CLOSE_FILE:
                compss.close_file(*parameters)
            elif command == DELETE_FILE:
                result = compss.delete_file(*parameters)
                out_queue.put(result)
            elif command == GET_FILE:
                compss.get_file(*parameters)
            elif command == GET_DIRECTORY:
                compss.get_directory(*parameters)
            elif command == BARRIER:
                compss.barrier(*parameters)
            elif command == BARRIER_GROUP:
                exception_message = compss.barrier_group(*parameters)
                out_queue.put(exception_message)
            elif command == OPEN_TASK_GROUP:
                compss.open_task_group(*parameters)
            elif command == CLOSE_TASK_GROUP:
                compss.close_task_group(*parameters)
            elif command == GET_LOGGING_PATH:
                log_path = compss.get_logging_path()
                out_queue.put(log_path)
            elif command == GET_NUMBER_OF_RESOURCES:
                num_resources = compss.get_number_of_resources(*parameters)
                out_queue.put(num_resources)
            elif command == REQUEST_RESOURCES:
                compss.request_resources(*parameters)
            elif command == FREE_RESOURCES:
                compss.free_resources(*parameters)
            elif command == REGISTER_CORE_ELEMENT:
                compss.register_core_element(*parameters)
            elif command == PROCESS_TASK:
                compss.process_task(*parameters)
            elif command == PROCESS_HTTP_TASK:
                compss.process_http_task(*parameters)
            elif command == SET_PIPES:
                compss.set_pipes(*parameters)
            elif command == READ_PIPES:
                compss.read_pipes(*parameters)
            elif command == SET_WALL_CLOCK:
                compss.set_wall_clock(*parameters)
            else:
                raise PyCOMPSsException("Unknown link command")
Exemplo n.º 9
0
def c_extension_link(in_queue, out_queue):
    # type: (..., ...) -> None
    """ Main C extension process.

    :param in_queue: Queue to receive messages.
    :param out_queue: Queue to send messages.
    :return: None
    """
    import compss

    alive = True
    while alive:
        message = in_queue.get()
        command = message[0]
        parameters = []
        if len(message) > 0:
            parameters = message[1:]
        if command == START:
            compss.start_runtime()
        elif command == SET_DEBUG:
            compss.set_debug(*parameters)
        elif command == STOP:
            compss.stop_runtime(*parameters)
            alive = False
        elif command == CANCEL_TASKS:
            compss.cancel_application_tasks(*parameters)
        elif command == ACCESSED_FILE:
            accessed = compss.accessed_file(*parameters)
            out_queue.put(accessed)
        elif command == OPEN_FILE:
            compss_name = compss.open_file(*parameters)
            out_queue.put(compss_name)
        elif command == CLOSE_FILE:
            compss.close_file(*parameters)
        elif command == DELETE_FILE:
            result = compss.delete_file(*parameters)
            out_queue.put(result)
        elif command == GET_FILE:
            compss.get_file(*parameters)
        elif command == GET_DIRECTORY:
            compss.get_directory(*parameters)
        elif command == BARRIER:
            compss.barrier(*parameters)
        elif command == BARRIER_GROUP:
            exception_message = compss.barrier_group(*parameters)
            out_queue.put(exception_message)
        elif command == OPEN_TASK_GROUP:
            compss.open_task_group(*parameters)
        elif command == CLOSE_TASK_GROUP:
            compss.close_task_group(*parameters)
        elif command == GET_LOGGING_PATH:
            log_path = compss.get_logging_path()
            out_queue.put(log_path)
        elif command == GET_NUMBER_OF_RESOURCES:
            num_resources = compss.get_number_of_resources(*parameters)
            out_queue.put(num_resources)
        elif command == REQUEST_RESOURCES:
            compss.request_resources(*parameters)
        elif command == FREE_RESOURCES:
            compss.free_resources(*parameters)
        elif command == REGISTER_CORE_ELEMENT:
            compss.register_core_element(*parameters)
        elif command == PROCESS_TASK:
            compss.process_task(*parameters)
        elif command == SET_PIPES:
            compss.set_pipes(*parameters)
        else:
            raise Exception("Unknown link command")
Exemplo n.º 10
0
def synchronize(obj, mode):
    """
    Synchronization function.
    This method retrieves the value of a future object.
    Calls the runtime in order to wait for the value and returns it when received.

    :param obj: Object to synchronize.
    :param mode: Direction of the object to synchronize.
    :return: The value of the object requested.
    """

    # TODO: Add a boolean to differentiate between files and object on the compss.get_file call. This change pretends
    # to obtain better traces. Must be implemented first in the Runtime, then in the bindings common C API and
    # finally add the boolean here
    global _current_id

    if is_psco(obj):
        obj_id = get_id(obj)
        if obj_id not in pending_to_synchronize:
            return obj
        else:
            # file_path is of the form storage://pscoId or file://sys_path_to_file
            file_path = compss.get_file("storage://" + str(obj_id), mode)
            # TODO: Add switch on protocol
            protocol, file_name = file_path.split('://')
            new_obj = get_by_id(file_name)
            return new_obj

    obj_id = get_object_id(obj)
    if obj_id not in pending_to_synchronize:
        return obj

    if __debug__:
        logger.debug("Synchronizing object %s with mode %s" % (obj_id, mode))

    file_name = objid_to_filename[obj_id]
    compss_file = compss.get_file(file_name, mode)

    # Runtime can return a path or a PSCOId
    if compss_file.startswith('/'):
        new_obj = deserialize_from_file(compss_file)
        compss.close_file(file_name, mode)
    else:
        new_obj = get_by_id(compss_file)
    new_obj_id = get_object_id(new_obj, True, True)

    # The main program won't work with the old object anymore, update mapping
    objid_to_filename[new_obj_id] = objid_to_filename[obj_id].replace(
        obj_id, new_obj_id)
    _objs_written_by_mp[new_obj_id] = objid_to_filename[new_obj_id]

    if __debug__:
        logger.debug("Deleting obj %s (new one is %s)" %
                     (str(obj_id), str(new_obj_id)))
    compss.delete_file(objid_to_filename[obj_id])
    objid_to_filename.pop(obj_id)
    pending_to_synchronize.pop(obj_id)

    if __debug__:
        logger.debug("Now object with id %s and %s has mapping %s" %
                     (new_obj_id, type(new_obj), file_name))

    return new_obj