示例#1
0
def launch_pycompss_application(
        app,
        func,
        log_level='off',  # type: str
        o_c=False,  # type: bool
        debug=False,  # type: bool
        graph=False,  # type: bool
        trace=False,  # type: bool
        monitor=None,  # type: int
        project_xml=None,  # type: str
        resources_xml=None,  # type: str
        summary=False,  # type: bool
        task_execution='compss',  # type: str
        storage_impl=None,  # type: str
        storage_conf=None,  # type: str
        streaming_backend=None,  # type: str
        streaming_master_name=None,  # type: str
        streaming_master_port=None,  # type: str
        task_count=50,  # type: int
        app_name=None,  # type: str
        uuid=None,  # type: str
        base_log_dir=None,  # type: str
        specific_log_dir=None,  # type: str
        extrae_cfg=None,  # type: str
        comm='NIO',  # type: str
        conn='es.bsc.compss.connectors.DefaultSSHConnector',  # noqa: E501
        master_name='',  # type: str
        master_port='',  # type: str
        scheduler='es.bsc.compss.scheduler.loadbalancing.LoadBalancingScheduler',  # noqa: E501
        jvm_workers='-Xms1024m,-Xmx1024m,-Xmn400m',
        cpu_affinity='automatic',  # type: str
        gpu_affinity='automatic',  # type: str
        fpga_affinity='automatic',  # type: str
        fpga_reprogram='',  # type: str
        profile_input='',  # type: str
        profile_output='',  # type: str
        scheduler_config='',  # type: str
        external_adaptation=False,  # type: bool
        propagate_virtual_environment=True,  # type: bool
        mpi_worker=False,  # type: bool
        *args,
        **kwargs):
    # type: (...) -> None
    """ Launch PyCOMPSs application from function.

    :param app: Application path
    :param func: Function
    :param log_level: Logging level [ 'trace'|'debug'|'info'|'api'|'off' ]
                      (default: 'off')
    :param o_c: Objects to string conversion [ True | False ] (default: False)
    :param debug: Debug mode [ True | False ] (default: False)
                  (overrides log_level)
    :param graph: Generate graph [ True | False ] (default: False)
    :param trace: Generate trace
                  [ True | False | 'scorep' | 'arm-map' | 'arm-ddt']
                  (default: False)
    :param monitor: Monitor refresh rate (default: None)
    :param project_xml: Project xml file path
    :param resources_xml: Resources xml file path
    :param summary: Execution summary [ True | False ] (default: False)
    :param task_execution: Task execution (default: 'compss')
    :param storage_impl: Storage implementation path
    :param storage_conf: Storage configuration file path
    :param streaming_backend: Streaming backend (default: None)
    :param streaming_master_name: Streaming master name (default: None)
    :param streaming_master_port: Streaming master port (default: None)
    :param task_count: Task count (default: 50)
    :param app_name: Application name (default: Interactive_date)
    :param uuid: UUId
    :param base_log_dir: Base logging directory
    :param specific_log_dir: Specific logging directory
    :param extrae_cfg: Extrae configuration file path
    :param comm: Communication library (default: NIO)
    :param conn: Connector (default: DefaultSSHConnector)
    :param master_name: Master Name (default: '')
    :param master_port: Master port (default: '')
    :param scheduler: Scheduler (default:
                  es.bsc.compss.scheduler.loadbalancing.LoadBalancingScheduler)
    :param jvm_workers: Java VM parameters
                        (default: '-Xms1024m,-Xmx1024m,-Xmn400m')
    :param cpu_affinity: CPU Core affinity (default: 'automatic')
    :param gpu_affinity: GPU Core affinity (default: 'automatic')
    :param fpga_affinity: FPA Core affinity (default: 'automatic')
    :param fpga_reprogram: FPGA repogram command (default: '')
    :param profile_input: Input profile  (default: '')
    :param profile_output: Output profile  (default: '')
    :param scheduler_config: Scheduler configuration  (default: '')
    :param external_adaptation: External adaptation [ True | False ]
                                (default: False)
    :param propagate_virtual_environment: Propagate virtual environment
                                          [ True | False ] (default: False)
    :param mpi_worker: Use the MPI worker [ True | False ] (default: False)
    :param args: Positional arguments
    :param kwargs: Named arguments
    :return: Execution result
    """
    # Let the Python binding know we are at master
    context.set_pycompss_context(context.MASTER)
    # Then we can import the appropriate start and stop functions from the API
    from pycompss.api.api import compss_start, compss_stop

    ##############################################################
    # INITIALIZATION
    ##############################################################

    # Initial dictionary with the user defined parameters
    all_vars = {
        'log_level': log_level,
        'debug': debug,
        'o_c': o_c,
        'graph': graph,
        'trace': trace,
        'monitor': monitor,
        'project_xml': project_xml,
        'resources_xml': resources_xml,
        'summary': summary,
        'task_execution': task_execution,
        'storage_impl': storage_impl,
        'storage_conf': storage_conf,
        'streaming_backend': streaming_backend,
        'streaming_master_name': streaming_master_name,
        'streaming_master_port': streaming_master_port,
        'task_count': task_count,
        'app_name': app_name,
        'uuid': uuid,
        'base_log_dir': base_log_dir,
        'specific_log_dir': specific_log_dir,
        'extrae_cfg': extrae_cfg,
        'comm': comm,
        'conn': conn,
        'master_name': master_name,
        'master_port': master_port,
        'scheduler': scheduler,
        'jvm_workers': jvm_workers,
        'cpu_affinity': cpu_affinity,
        'gpu_affinity': gpu_affinity,
        'fpga_affinity': fpga_affinity,
        'fpga_reprogram': fpga_reprogram,
        'profile_input': profile_input,
        'profile_output': profile_output,
        'scheduler_config': scheduler_config,
        'external_adaptation': external_adaptation,
        'propagate_virtual_environment': propagate_virtual_environment,
        'mpi_worker': mpi_worker
    }

    # Check the provided flags
    flags, issues = check_flags(all_vars)
    if not flags:
        print_flag_issues(issues)
        return None

    # Prepare the environment
    env_vars = prepare_environment(False, o_c, storage_impl, app, debug, trace,
                                   mpi_worker)
    all_vars.update(env_vars)

    monitoring_vars = prepare_loglevel_graph_for_monitoring(
        monitor, graph, debug, log_level)
    all_vars.update(monitoring_vars)

    if RUNNING_IN_SUPERCOMPUTER:
        updated_vars = updated_variables_in_sc()
        all_vars.update(updated_vars)

    to_update = prepare_tracing_environment(all_vars['trace'],
                                            all_vars['extrae_lib'],
                                            all_vars['ld_library_path'])
    all_vars['trace'], all_vars['ld_library_path'] = to_update

    inf_vars = check_infrastructure_variables(all_vars['project_xml'],
                                              all_vars['resources_xml'],
                                              all_vars['compss_home'],
                                              all_vars['app_name'],
                                              all_vars['file_name'],
                                              all_vars['external_adaptation'])
    all_vars.update(inf_vars)

    create_init_config_file(**all_vars)

    ##############################################################
    # RUNTIME START
    ##############################################################

    # Runtime start
    compss_start(log_level, all_vars['trace'], True)

    # Setup logging
    binding_log_path = get_log_path()
    log_path = os.path.join(all_vars['compss_home'], 'Bindings', 'python',
                            str(all_vars['major_version']), 'log')
    set_temporary_directory(binding_log_path)
    logging_cfg_file = get_logging_cfg_file(log_level)
    init_logging(os.path.join(log_path, logging_cfg_file), binding_log_path)
    logger = logging.getLogger("pycompss.runtime.launch")

    logger.debug('--- START ---')
    logger.debug('PyCOMPSs Log path: %s' % log_path)

    logger.debug("Starting storage")
    persistent_storage = master_init_storage(all_vars['storage_conf'], logger)

    logger.debug("Starting streaming")
    streaming = init_streaming(all_vars['streaming_backend'],
                               all_vars['streaming_master_name'],
                               all_vars['streaming_master_port'])

    saved_argv = sys.argv
    sys.argv = args
    # Execution:
    with event(APPLICATION_RUNNING_EVENT, master=True):
        if func is None or func == '__main__':
            if IS_PYTHON3:
                exec(open(app).read())
            else:
                execfile(app)  # noqa
            result = None
        else:
            if IS_PYTHON3:
                import importlib.util
                spec = importlib.util.spec_from_file_location(
                    all_vars['file_name'], app)  # noqa: E501
                imported_module = importlib.util.module_from_spec(spec)
                spec.loader.exec_module(imported_module)  # noqa
            else:
                import imp  # noqa
                imported_module = imp.load_source(all_vars['file_name'],
                                                  app)  # noqa
            method_to_call = getattr(imported_module, func)
            result = method_to_call(*args, **kwargs)
    # Recover the system arguments
    sys.argv = saved_argv

    # Stop streaming
    if streaming:
        stop_streaming()

    # Stop persistent storage
    if persistent_storage:
        master_stop_storage(logger)

    logger.debug('--- END ---')

    ##############################################################
    # RUNTIME STOP
    ##############################################################

    # Stop runtime
    compss_stop()

    return result
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#



'''
@author: etejedor
'''

app_path = None

if __name__ == "__main__":
    import sys
    from pycompss.api.api import compss_start, compss_stop
    
    compss_start()
    
    # remove launch.py from sys.argv, it will be inherited by the app through execfile
    sys.argv = sys.argv[1:]
    app_path = sys.argv[0]
    
    execfile(app_path)
    
    compss_stop()
示例#3
0
def compss_main():
    # type: () -> None
    """ PyCOMPSs main function.

    General call:
    python $PYCOMPSS_HOME/pycompss/runtime/launch.py $log_level
           $PyObject_serialize $storage_conf $streaming_backend
           $streaming_master_name $streaming_master_port
           $fullAppPath $application_args

    :return: None
    """
    global APP_PATH

    # Let the Python binding know we are at master
    context.set_pycompss_context(context.MASTER)
    # Then we can import the appropriate start and stop functions from the API
    from pycompss.api.api import compss_start, compss_stop

    # See parse_arguments, defined above
    # In order to avoid parsing user arguments, we are going to remove user
    # args from sys.argv
    user_sys_argv = sys.argv[9:]
    sys.argv = sys.argv[:9]
    args = parse_arguments()
    # We are done, now sys.argv must contain user args only
    sys.argv = [args.app_path] + user_sys_argv

    # Get log_level
    log_level = args.log_level

    # Setup tracing
    tracing = int(args.tracing)

    # Start the runtime
    compss_start(log_level, tracing, False)

    # Get object_conversion boolean
    set_object_conversion(args.object_conversion == 'true')

    # Get storage configuration at master
    storage_conf = args.storage_configuration

    # Get application execution path
    APP_PATH = args.app_path

    # Setup logging
    binding_log_path = get_log_path()
    log_path = os.path.join(os.getenv('COMPSS_HOME'), 'Bindings', 'python',
                            str(_PYTHON_VERSION), 'log')
    set_temporary_directory(binding_log_path)
    logging_cfg_file = get_logging_cfg_file(log_level)
    init_logging(os.path.join(log_path, logging_cfg_file), binding_log_path)
    logger = logging.getLogger("pycompss.runtime.launch")

    # Get JVM options
    # jvm_opts = os.environ['JVM_OPTIONS_FILE']
    # from pycompss.util.jvm.parser import convert_to_dict
    # opts = convert_to_dict(jvm_opts)
    # storage_conf = opts.get('-Dcompss.storage.conf')

    exit_code = 0
    try:
        if __debug__:
            logger.debug('--- START ---')
            logger.debug('PyCOMPSs Log path: %s' % binding_log_path)

        # Start persistent storage
        persistent_storage = master_init_storage(storage_conf, logger)

        # Start streaming
        streaming = init_streaming(args.streaming_backend,
                                   args.streaming_master_name,
                                   args.streaming_master_port)

        # Show module warnings
        if __debug__:
            show_optional_module_warnings()

        # MAIN EXECUTION
        with event(APPLICATION_RUNNING_EVENT, master=True):
            if IS_PYTHON3:
                with open(APP_PATH) as f:
                    exec(compile(f.read(), APP_PATH, 'exec'), globals())
            else:
                execfile(APP_PATH, globals())  # MAIN EXECUTION

        # Stop streaming
        if streaming:
            stop_streaming()

        # Stop persistent storage
        if persistent_storage:
            master_stop_storage(logger)

        # End
        if __debug__:
            logger.debug('--- END ---')
    except SystemExit as e:
        if e.code != 0:
            print('[ ERROR ]: User program ended with exitcode %s.' % e.code)
            print('\t\tShutting down runtime...')
            exit_code = e.code
    except SerializerException:
        exit_code = 1
        # If an object that can not be serialized has been used as a parameter.
        print("[ ERROR ]: Serialization exception")
        exc_type, exc_value, exc_traceback = sys.exc_info()
        lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
        for line in lines:
            if APP_PATH in line:
                print('[ ERROR ]: In: %s', line)
        exit_code = 1
    except COMPSsException as e:
        # Any other exception occurred
        print("[ ERROR ]: A COMPSs exception occurred: " + str(e))
        traceback.print_exc()
        exit_code = 0  # COMPSs exception is not considered an error
    except Exception as e:
        # Any other exception occurred
        print("[ ERROR ]: An exception occurred: " + str(e))
        traceback.print_exc()
        exit_code = 1
    finally:
        # Stop runtime
        compss_stop(exit_code)
        sys.stdout.flush()
        sys.stderr.flush()
        sys.exit(exit_code)
示例#4
0
def start(
        log_level='off',
        o_c=False,
        debug=False,
        graph=False,
        trace=False,
        monitor=None,
        project_xml=None,
        resources_xml=None,
        summary=False,
        taskExecution='compss',
        storageConf=None,
        taskCount=50,
        appName='Interactive',
        uuid=None,
        baseLogDir=None,
        specificLogDir=None,
        extraeCfg=None,
        comm='NIO',
        conn='es.bsc.compss.connectors.DefaultSSHConnector',
        masterName='',
        masterPort='',
        scheduler='es.bsc.compss.scheduler.loadBalancingScheduler.LoadBalancingScheduler',
        jvmWorkers='-Xms1024m,-Xmx1024m,-Xmn400m',
        cpuAffinity='automatic',
        gpuAffinity='automatic',
        profileInput='',
        profileOutput='',
        scheduler_config='',
        external_adaptation=False,
        propagate_virtual_environment=False,
        verbose=False):
    launchPath = os.path.dirname(os.path.realpath(__file__))
    # compss_home = launchPath without the last 4 folders:
    # Bindings/python/version/pycompss
    compss_home = os.path.sep.join(launchPath.split(os.path.sep)[:-4])
    os.environ['COMPSS_HOME'] = compss_home

    # Get environment variables
    cp = os.getcwd() + '/'
    pythonPath = os.environ['PYTHONPATH']
    classPath = os.environ['CLASSPATH']
    ld_library_path = os.environ['LD_LIBRARY_PATH']

    # Set extrae dependencies
    extrae_home = compss_home + '/Dependencies/extrae'
    extrae_lib = extrae_home + '/lib'
    os.environ['EXTRAE_HOME'] = extrae_home
    os.environ['LD_LIBRARY_PATH'] = extrae_lib + ':' + ld_library_path

    if trace is False:
        trace = 0
    elif trace == 'basic' or trace is True:
        trace = 1
        os.environ['LD_PRELOAD'] = extrae_lib + '/libpttrace.so'
    elif trace == 'advanced':
        trace = 2
        os.environ['LD_PRELOAD'] = extrae_lib + '/libpttrace.so'
    else:
        print('ERROR: Wrong tracing parameter ( [ True | basic ] | \
               advanced | False)')
        return -1

    if monitor is not None:
        # Enable the graph if the monitoring is enabled
        graph = True

    global graphing
    graphing = graph
    global tracing
    tracing = trace
    global monitoring
    monitoring = monitor

    __export_globals__()

    print("********************************************************")
    print("*************** PyCOMPSs Interactive *******************")
    print("********************************************************")
    print("*          .-~~-.--.                ____        ____   *")
    print("*         :         )              |___ \      |___ \  *")
    print("*   .~ ~ -.\       /.- ~~ .          __) |       __) | *")
    print("*   >       `.   .'       <         / __/   _   / __/  *")
    print("*  (         .- -.         )       |_____| |_| |_____| *")
    print("*   `- -.-~  `- -'  ~-.- -'                            *")
    print("*     (        :        )           _ _ .-:            *")
    print("*      ~--.    :    .--~        .-~  .-~  }            *")
    print("*          ~-.-^-.-~ \_      .~  .-~   .~              *")
    print("*                   \ \ '     \ '_ _ -~                *")
    print("*                    \`.\`.    //                      *")
    print("*           . - ~ ~-.__\`.\`-.//                       *")
    print("*       .-~   . - ~  }~ ~ ~-.~-.                       *")
    print("*     .' .-~      .-~       :/~-.~-./:                 *")
    print("*    /_~_ _ . - ~                 ~-.~-._              *")
    print("*                                     ~-.<             *")
    print("********************************************************")

    # print("*          .-~~-.--.                ____       _____   *")
    # print("*         :         )              |___ \     |___ /   *")
    # print("*   .~ ~ -.\       /.- ~~ .          __) |      |_ \   *")
    # print("*   >       `.   .'       <         / __/   _   __) |  *")
    # print("*  (         .- -.         )       |_____| |_| |___/   *")

    ##############################################################
    # INITIALIZATION
    ##############################################################

    # Build a dictionary with all variables needed for initializing the runtime
    config = {}
    config['compss_home'] = compss_home
    config['debug'] = debug
    if project_xml is None:
        projPath = 'Runtime/configuration/xml/projects/default_project.xml'
        config['project_xml'] = compss_home + os.path.sep + projPath
    else:
        config['project_xml'] = project_xml
    if resources_xml is None:
        resPath = 'Runtime/configuration/xml/resources/default_resources.xml'
        config['resources_xml'] = compss_home + os.path.sep + resPath
    else:
        config['resources_xml'] = resources_xml
    config['summary'] = summary
    config['taskExecution'] = taskExecution
    config['storageConf'] = storageConf
    config['taskCount'] = taskCount
    if appName is None:
        config['appName'] = 'Interactive'
    else:
        config['appName'] = appName
    config['uuid'] = uuid
    config['baseLogDir'] = baseLogDir
    config['specificLogDir'] = specificLogDir
    config['graph'] = graph
    config['monitor'] = monitor
    config['trace'] = trace
    config['extraeCfg'] = extraeCfg
    config['comm'] = comm
    config['conn'] = conn
    config['masterName'] = masterName
    config['masterPort'] = masterPort
    config['scheduler'] = scheduler
    config['cp'] = cp
    config['classpath'] = classPath
    config['jvmWorkers'] = jvmWorkers
    config['pythonPath'] = pythonPath
    config['cpuAffinity'] = cpuAffinity
    config['gpuAffinity'] = gpuAffinity
    config['profileInput'] = profileInput
    config['profileOutput'] = profileOutput
    config['scheduler_config'] = scheduler_config
    if external_adaptation:
        config['external_adaptation'] = 'true'
    else:
        config['external_adaptation'] = 'false'

    major_version = sys.version_info[0]
    python_interpreter = 'python' + str(major_version)
    config['python_interpreter'] = python_interpreter
    config['python_version'] = str(major_version)

    if 'VIRTUAL_ENV' in os.environ:
        # Running within a virtual environment
        python_virtual_environment = os.environ['VIRTUAL_ENV']
    else:
        python_virtual_environment = 'null'
    config['python_virtual_environment'] = python_virtual_environment
    config[
        'python_propagate_virtual_environment'] = propagate_virtual_environment

    initialize_compss(config)

    ##############################################################
    # RUNTIME START
    ##############################################################

    print("* - Starting COMPSs runtime...                         *")
    compss_start()

    if o_c is True:
        # set cross-module variable
        binding.object_conversion = True
    else:
        # set cross-module variable
        binding.object_conversion = False

    # Remove launch.py, log_level and object_conversion from sys.argv,
    # It will be inherited by the app through execfile
    # sys.argv = sys.argv[3:]
    # Get application execution path
    # app_path = sys.argv[0]  # not needed in interactive mode

    global log_path
    log_path = get_log_path()
    binding.temp_dir = mkdtemp(prefix='pycompss', dir=log_path + '/tmpFiles/')
    print("* - Log path : " + log_path)

    # Logging setup
    if log_level == "debug":
        jsonPath = '/Bindings/python/' + str(
            major_version) + '/log/logging.json.debug'
        init_logging(os.getenv('COMPSS_HOME') + jsonPath, log_path)
    elif log_level == "info":
        jsonPath = '/Bindings/python/' + str(
            major_version) + '/log/logging.json.off'
        init_logging(os.getenv('COMPSS_HOME') + jsonPath, log_path)
    elif log_level == "off":
        jsonPath = '/Bindings/python/' + str(
            major_version) + '/log/logging.json.off'
        init_logging(os.getenv('COMPSS_HOME') + jsonPath, log_path)
    else:
        # Default
        jsonPath = '/Bindings/python/' + str(
            major_version) + '/log/logging.json'
        init_logging(os.getenv('COMPSS_HOME') + jsonPath, log_path)
    logger = logging.getLogger("pycompss.runtime.launch")

    printSetup(verbose, log_level, o_c, debug, graph, trace, monitor,
               project_xml, resources_xml, summary, taskExecution, storageConf,
               taskCount, appName, uuid, baseLogDir, specificLogDir, extraeCfg,
               comm, conn, masterName, masterPort, scheduler, jvmWorkers,
               cpuAffinity, gpuAffinity, profileInput, profileOutput,
               scheduler_config, external_adaptation, python_interpreter,
               major_version, python_virtual_environment,
               propagate_virtual_environment)

    logger.debug("--- START ---")
    logger.debug("PyCOMPSs Log path: %s" % log_path)
    if storageConf is not None:
        logger.debug("Storage configuration file: %s" % storageConf)
        from storage.api import init as initStorage
        initStorage(config_file_path=storageConf)
        global persistent_storage
        persistent_storage = True

    # MAIN EXECUTION
    # let the user write an interactive application
    print("* - PyCOMPSs Runtime started... Have fun!              *")
    print("********************************************************")
示例#5
0
def start(
        log_level='off',  # type: str
        debug=False,  # type: bool
        o_c=False,  # type: bool
        graph=False,  # type: bool
        trace=False,  # type: bool
        monitor=None,  # type: int
        project_xml=None,  # type: str
        resources_xml=None,  # type: str
        summary=False,  # type: bool
        task_execution='compss',  # type: str
        storage_impl=None,  # type: str
        storage_conf=None,  # type: str
        streaming_backend=None,  # type: str
        streaming_master_name=None,  # type: str
        streaming_master_port=None,  # type: str
        task_count=50,  # type: int
        app_name=INTERACTIVE_FILE_NAME,  # type: str
        uuid=None,  # type: str
        base_log_dir=None,  # type: str
        specific_log_dir=None,  # type: str
        extrae_cfg=None,  # type: str
        comm='NIO',  # type: str
        conn='es.bsc.compss.connectors.DefaultSSHConnector',  # type: str
        master_name='',  # type: str
        master_port='',  # type: str
        scheduler='es.bsc.compss.scheduler.loadbalancing.LoadBalancingScheduler',  # type: str  # noqa: E501
        jvm_workers='-Xms1024m,-Xmx1024m,-Xmn400m',  # type: str
        cpu_affinity='automatic',  # type: str
        gpu_affinity='automatic',  # type: str
        fpga_affinity='automatic',  # type: str
        fpga_reprogram='',  # type: str
        profile_input='',  # type: str
        profile_output='',  # type: str
        scheduler_config='',  # type: str
        external_adaptation=False,  # type: bool
        propagate_virtual_environment=True,  # type: bool
        mpi_worker=False,  # type: bool
        verbose=False  # type: bool
):
    # type: (...) -> None
    """ Start the runtime in interactive mode.

    :param log_level: Logging level [ 'trace'|'debug'|'info'|'api'|'off' ]
                      (default: 'off')
    :param debug: Debug mode [ True | False ]
                  (default: False) (overrides log-level)
    :param o_c: Objects to string conversion [ True|False ]
                (default: False)
    :param graph: Generate graph [ True|False ]
                  (default: False)
    :param trace: Generate trace [ True|False|'scorep'|'arm-map'|'arm-ddt' ]
                  (default: False)
    :param monitor: Monitor refresh rate
                    (default: None)
    :param project_xml: Project xml file path
                        (default: None)
    :param resources_xml: Resources xml file path
                          (default: None)
    :param summary: Execution summary [ True | False ]
                    (default: False)
    :param task_execution: Task execution
                           (default: 'compss')
    :param storage_impl: Storage implementation path
                         (default: None)
    :param storage_conf: Storage configuration file path
                         (default: None)
    :param streaming_backend: Streaming backend
                              (default: None)
    :param streaming_master_name: Streaming master name
                                  (default: None)
    :param streaming_master_port: Streaming master port
                                  (default: None)
    :param task_count: Task count
                       (default: 50)
    :param app_name: Application name
                     default: INTERACTIVE_FILE_NAME)
    :param uuid: UUId
                 (default: None)
    :param base_log_dir: Base logging directory
                         (default: None)
    :param specific_log_dir: Specific logging directory
                             (default: None)
    :param extrae_cfg: Extrae configuration file path
                       (default: None)
    :param comm: Communication library
                 (default: NIO)
    :param conn: Connector
                 (default: DefaultSSHConnector)
    :param master_name: Master Name
                        (default: '')
    :param master_port: Master port
                        (default: '')
    :param scheduler: Scheduler (see runcompss)
                      (default: es.bsc.compss.scheduler.loadbalancing.LoadBalancingScheduler)  # noqa
    :param jvm_workers: Java VM parameters
                        (default: '-Xms1024m,-Xmx1024m,-Xmn400m')
    :param cpu_affinity: CPU Core affinity
                         (default: 'automatic')
    :param gpu_affinity: GPU affinity
                         (default: 'automatic')
    :param fpga_affinity: FPGA affinity
                          (default: 'automatic')
    :param fpga_reprogram: FPGA repogram command
                           (default: '')
    :param profile_input: Input profile
                          (default: '')
    :param profile_output: Output profile
                           (default: '')
    :param scheduler_config: Scheduler configuration
                             (default: '')
    :param external_adaptation: External adaptation [ True|False ]
                                (default: False)
    :param propagate_virtual_environment: Propagate virtual environment [ True|False ]  # noqa
                                          (default: False)
    :param mpi_worker: Use the MPI worker [ True|False ]
                       (default: False)
    :param verbose: Verbose mode [ True|False ]
                    (default: False)
    :return: None
    """
    # Export global variables
    global GRAPHING
    GRAPHING = graph
    __export_globals__()

    interactive_helpers.DEBUG = debug

    __show_flower__()

    # Let the Python binding know we are at master
    context.set_pycompss_context(context.MASTER)
    # Then we can import the appropriate start and stop functions from the API
    from pycompss.api.api import compss_start

    ##############################################################
    # INITIALIZATION
    ##############################################################

    # Initial dictionary with the user defined parameters
    all_vars = {
        'log_level': log_level,
        'debug': debug,
        'o_c': o_c,
        'graph': graph,
        'trace': trace,
        'monitor': monitor,
        'project_xml': project_xml,
        'resources_xml': resources_xml,
        'summary': summary,
        'task_execution': task_execution,
        'storage_impl': storage_impl,
        'storage_conf': storage_conf,
        'streaming_backend': streaming_backend,
        'streaming_master_name': streaming_master_name,
        'streaming_master_port': streaming_master_port,
        'task_count': task_count,
        'app_name': app_name,
        'uuid': uuid,
        'base_log_dir': base_log_dir,
        'specific_log_dir': specific_log_dir,
        'extrae_cfg': extrae_cfg,
        'comm': comm,
        'conn': conn,
        'master_name': master_name,
        'master_port': master_port,
        'scheduler': scheduler,
        'jvm_workers': jvm_workers,
        'cpu_affinity': cpu_affinity,
        'gpu_affinity': gpu_affinity,
        'fpga_affinity': fpga_affinity,
        'fpga_reprogram': fpga_reprogram,
        'profile_input': profile_input,
        'profile_output': profile_output,
        'scheduler_config': scheduler_config,
        'external_adaptation': external_adaptation,
        'propagate_virtual_environment': propagate_virtual_environment,
        'mpi_worker': mpi_worker
    }

    # Check the provided flags
    flags, issues = check_flags(all_vars)
    if not flags:
        print_flag_issues(issues)
        return None

    # Prepare the environment
    env_vars = prepare_environment(True, o_c, storage_impl, 'undefined', debug,
                                   trace, mpi_worker)
    all_vars.update(env_vars)

    # Update the log level and graph values if monitoring is enabled
    monitoring_vars = prepare_loglevel_graph_for_monitoring(
        monitor, graph, debug, log_level)
    all_vars.update(monitoring_vars)

    # Check if running in supercomputer and update the variables accordingly
    # with the defined in the launcher and exported in environment variables.
    if RUNNING_IN_SUPERCOMPUTER:
        updated_vars = updated_variables_in_sc()
        if verbose:
            print("- Overridden project xml with: %s" %
                  updated_vars['project_xml'])
            print("- Overridden resources xml with: %s" %
                  updated_vars['resources_xml'])
            print("- Overridden master name with: %s" %
                  updated_vars['master_name'])
            print("- Overridden master port with: %s" %
                  updated_vars['master_port'])
            print("- Overridden uuid with: %s" % updated_vars['uuid'])
            print("- Overridden base log dir with: %s" %
                  updated_vars['base_log_dir'])
            print("- Overridden specific log dir with: %s" %
                  updated_vars['specific_log_dir'])
            print("- Overridden storage conf with: %s" %
                  updated_vars['storage_conf'])
            print("- Overridden log level with: %s" %
                  str(updated_vars['log_level']))
            print("- Overridden debug with: %s" % str(updated_vars['debug']))
            print("- Overridden trace with: %s" % str(updated_vars['trace']))
        all_vars.update(updated_vars)

    # Update the tracing environment if set and set the appropriate trace
    # integer value
    tracing_vars = prepare_tracing_environment(all_vars['trace'],
                                               all_vars['extrae_lib'],
                                               all_vars['ld_library_path'])
    all_vars['trace'], all_vars['ld_library_path'] = tracing_vars

    # Update the infrastructure variables if necessary
    inf_vars = check_infrastructure_variables(all_vars['project_xml'],
                                              all_vars['resources_xml'],
                                              all_vars['compss_home'],
                                              all_vars['app_name'],
                                              all_vars['file_name'],
                                              all_vars['external_adaptation'])
    all_vars.update(inf_vars)

    # With all this information, create the configuration file for the
    # runtime start
    create_init_config_file(**all_vars)

    ##############################################################
    # RUNTIME START
    ##############################################################

    print("* - Starting COMPSs runtime...                       *")
    sys.stdout.flush()  # Force flush
    compss_start(log_level, all_vars['trace'], True)

    global LOG_PATH
    LOG_PATH = get_log_path()
    set_temporary_directory(LOG_PATH)
    print("* - Log path : " + LOG_PATH)

    # Setup logging
    binding_log_path = get_log_path()
    log_path = os.path.join(all_vars['compss_home'], 'Bindings', 'python',
                            str(all_vars['major_version']), 'log')
    set_temporary_directory(binding_log_path)
    logging_cfg_file = get_logging_cfg_file(log_level)
    init_logging(os.path.join(log_path, logging_cfg_file), binding_log_path)
    logger = logging.getLogger("pycompss.runtime.launch")

    __print_setup__(verbose, all_vars)

    logger.debug("--- START ---")
    logger.debug("PyCOMPSs Log path: %s" % LOG_PATH)

    logger.debug("Starting storage")
    global PERSISTENT_STORAGE
    PERSISTENT_STORAGE = master_init_storage(all_vars['storage_conf'], logger)

    logger.debug("Starting streaming")
    global STREAMING
    STREAMING = init_streaming(all_vars['streaming_backend'],
                               all_vars['streaming_master_name'],
                               all_vars['streaming_master_port'])

    # MAIN EXECUTION
    # let the user write an interactive application
    print("* - PyCOMPSs Runtime started... Have fun!            *")
    print("******************************************************")

    # Emit the application start event (the 0 is in the stop function)
    emit_manual_event(APPLICATION_RUNNING_EVENT)
示例#6
0
def launch_pycompss_application(
        app,
        func,
        args=[],
        kwargs={},
        log_level='off',
        o_c=False,
        debug=False,
        graph=False,
        trace=False,
        monitor=None,
        project_xml=None,
        resources_xml=None,
        summary=False,
        taskExecution='compss',
        storageConf=None,
        taskCount=50,
        appName=None,
        uuid=None,
        baseLogDir=None,
        specificLogDir=None,
        extraeCfg=None,
        comm='NIO',
        conn='es.bsc.compss.connectors.DefaultSSHConnector',
        masterName='',
        masterPort='',
        scheduler='es.bsc.compss.scheduler.loadBalancingScheduler.LoadBalancingScheduler',
        jvmWorkers='-Xms1024m,-Xmx1024m,-Xmn400m',
        obj_conv=False,
        cpuAffinity='automatic',
        gpuAffinity='automatic',
        profileInput='',
        profileOutput='',
        scheduler_config='',
        external_adaptation=False,
        python_propagate_virtual_environment=True):
    global app_path
    launchPath = os.path.dirname(os.path.abspath(__file__))
    # compss_home = launchPath without the last 4 folders:
    # (Bindings/python/pycompss/runtime)
    compss_home = os.path.sep.join(launchPath.split(os.path.sep)[:-4])

    # Grab the existing PYTHONPATH and CLASSPATH values
    pythonpath = os.environ['PYTHONPATH']
    classpath = os.environ['CLASSPATH']

    # Enable/Disable object to string conversion
    binding.object_conversion = obj_conv

    # Get the filename and its path.
    file_name = os.path.splitext(os.path.basename(app))[0]
    cp = os.path.dirname(app)

    # Build a dictionary with all variables needed for initializing the runtime.
    config = {}
    config['compss_home'] = compss_home
    config['debug'] = debug
    if project_xml is None:
        projXml = 'Runtime/configuration/xml/projects/default_project.xml'
        config['project_xml'] = compss_home + os.path.sep + projXml
    else:
        config['project_xml'] = project_xml
    if resources_xml is None:
        resXml = 'Runtime/configuration/xml/resources/default_resources.xml'
        config['resources_xml'] = compss_home + os.path.sep + resXml
    else:
        config['resources_xml'] = resources_xml
    config['summary'] = summary
    config['taskExecution'] = taskExecution
    config['storageConf'] = storageConf
    config['taskCount'] = taskCount
    if appName is None:
        config['appName'] = file_name
    else:
        config['appName'] = appName
    config['uuid'] = uuid
    config['baseLogDir'] = baseLogDir
    config['specificLogDir'] = specificLogDir
    config['graph'] = graph
    config['monitor'] = monitor
    config['trace'] = trace
    config['extraeCfg'] = extraeCfg
    config['comm'] = comm
    config['conn'] = conn
    config['masterName'] = masterName
    config['masterPort'] = masterPort
    config['scheduler'] = scheduler
    config['cp'] = cp
    config['classpath'] = classpath
    config['jvmWorkers'] = jvmWorkers
    config['pythonPath'] = pythonpath
    config['cpuAffinity'] = cpuAffinity
    config['gpuAffinity'] = gpuAffinity
    config['profileInput'] = profileInput
    config['profileOutput'] = profileOutput
    config['scheduler_config'] = scheduler_config
    if external_adaptation:
        config['external_adaptation'] = 'true'
    else:
        config['external_adaptation'] = 'false'
    config['python_interpreter'] = 'python' + str(sys.version_info[0])
    config['python_version'] = str(sys.version_info[0])
    if 'VIRTUAL_ENV' in os.environ:
        # Running within a virtual environment
        python_virtual_environment = os.environ['VIRTUAL_ENV']
    else:
        python_virtual_environment = 'null'
    config['python_virtual_environment'] = python_virtual_environment
    config[
        'python_propagate_virtual_environment'] = python_propagate_virtual_environment

    initialize_compss(config)

    # Runtime start
    compss_start()

    # Configure logging
    app_path = app
    log_path = get_log_path()
    if debug:
        # DEBUG
        init_logging(compss_home + '/Bindings/python/log/logging.json.debug',
                     log_path)
    else:
        # NO DEBUG
        init_logging(compss_home + '/Bindings/python/log/logging.json',
                     log_path)
    logger = logging.getLogger("pycompss.runtime.launch")

    logger.debug('--- START ---')
    logger.debug('PyCOMPSs Log path: %s' % log_path)
    saved_argv = sys.argv
    sys.argv = args
    # Execution:
    if func is None or func == '__main__':
        result = execfile(app)
    else:
        import imp
        imported_module = imp.load_source(file_name, app)
        methodToCall = getattr(imported_module, func)
        result = methodToCall(*args, **kwargs)
    # Recover the system arguments
    sys.argv = saved_argv
    logger.debug('--- END ---')

    compss_stop()

    return result
示例#7
0
def compss_main():
    '''
    General call:
    python $PYCOMPSS_HOME/pycompss/runtime/launch.py $log_level $PyObject_serialize $storageConf $fullAppPath $application_args
    '''
    global app_path

    # Start the runtime, see bindings commons
    compss_start()

    # See parse_arguments, defined above
    # In order to avoid parsing user arguments, we are going to remove user
    # args from sys.argv
    user_sys_argv = sys.argv[5:]
    sys.argv = sys.argv[:5]
    args = parse_arguments()
    # We are done, now sys.argv must contain user args only
    sys.argv = [args.app_path] + user_sys_argv

    # Get log_level
    log_level = args.log_level

    # Get object_conversion boolean
    binding.object_conversion = args.object_conversion == 'true'

    # Get storage configuration at master
    storage_conf = args.storage_configuration
    persistent_storage = False
    if storage_conf != 'null':
        persistent_storage = True
        from storage.api import init as init_storage
        from storage.api import finish as finish_storage

    # Get application execution path
    app_path = args.app_path

    binding_log_path = get_log_path()
    log_path = os.path.join(os.getenv('COMPSS_HOME'), 'Bindings', 'python',
                            str(py_version), 'log')
    binding.temp_dir = mkdtemp(prefix='pycompss',
                               dir=os.path.join(binding_log_path, 'tmpFiles/'))

    logging_cfg_file = get_logging_cfg_file(log_level)

    init_logging(os.path.join(log_path, logging_cfg_file), binding_log_path)
    if __debug__:
        logger = logging.getLogger('pycompss.runtime.launch')

    # Get JVM options
    jvm_opts = os.environ['JVM_OPTIONS_FILE']
    opts = convert_to_dict(jvm_opts)
    # storage_conf = opts.get('-Dcompss.storage.conf')

    try:
        if __debug__:
            logger.debug('--- START ---')
            logger.debug('PyCOMPSs Log path: %s' % binding_log_path)
        if persistent_storage:
            if __debug__:
                logger.debug('Storage configuration file: %s' % storage_conf)
            init_storage(config_file_path=storage_conf)
        show_optional_module_warnings()
        # MAIN EXECUTION
        if sys.version_info >= (3, 0):
            exec(compile(open(app_path).read(), app_path, 'exec'), globals())
        else:
            execfile(app_path, globals())  # MAIN EXECUTION
        if persistent_storage:
            finish_storage()
        if __debug__:
            logger.debug('--- END ---')
    except SystemExit as e:
        if e.code != 0:  # Seems this is not happening
            print('[ ERROR ]: User program ended with exitcode %s.' % e.code)
            print('\t\tShutting down runtime...')
    except SerializerException:
        # If an object that can not be serialized has been used as a parameter.
        exc_type, exc_value, exc_traceback = sys.exc_info()
        lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
        for line in lines:
            if app_path in line:
                print('[ ERROR ]: In: %s', line)
    finally:
        compss_stop()
        sys.stdout.flush()
        sys.stderr.flush()
示例#8
0
def start(
        log_level="off",  # type: str
        debug=False,  # type: bool
        o_c=False,  # type: bool
        graph=False,  # type: bool
        trace=False,  # type: bool
        monitor=-1,  # type: int
        project_xml="",  # type: str
        resources_xml="",  # type: str
        summary=False,  # type: bool
        task_execution="compss",  # type: str
        storage_impl="",  # type: str
        storage_conf="",  # type: str
        streaming_backend="",  # type: str
        streaming_master_name="",  # type: str
        streaming_master_port="",  # type: str
        task_count=50,  # type: int
        app_name=INTERACTIVE_FILE_NAME,  # type: str
        uuid="",  # type: str
        base_log_dir="",  # type: str
        specific_log_dir="",  # type: str
        extrae_cfg="",  # type: str
        comm="NIO",  # type: str
        conn=DEFAULT_CONN,  # type: str
        master_name="",  # type: str
        master_port="",  # type: str
        scheduler=DEFAULT_SCHED,  # type: str
        jvm_workers=DEFAULT_JVM_WORKERS,  # type: str
        cpu_affinity="automatic",  # type: str
        gpu_affinity="automatic",  # type: str
        fpga_affinity="automatic",  # type: str
        fpga_reprogram="",  # type: str
        profile_input="",  # type: str
        profile_output="",  # type: str
        scheduler_config="",  # type: str
        external_adaptation=False,  # type: bool
        propagate_virtual_environment=True,  # type: bool
        mpi_worker=False,  # type: bool
        worker_cache=False,  # type: typing.Union[bool, str]
        shutdown_in_node_failure=False,  # type: bool
        io_executors=0,  # type: int
        env_script="",  # type: str
        reuse_on_block=True,  # type: bool
        nested_enabled=False,  # type: bool
        tracing_task_dependencies=False,  # type: bool
        trace_label="",  # type: str
        extrae_cfg_python="",  # type: str
        wcl=0,  # type: int
        cache_profiler=False,  # type: bool
        verbose=False  # type: bool
):  # type: (...) -> None
    """ Start the runtime in interactive mode.

    :param log_level: Logging level [ "trace"|"debug"|"info"|"api"|"off" ]
                      (default: "off")
    :param debug: Debug mode [ True | False ]
                  (default: False) (overrides log-level)
    :param o_c: Objects to string conversion [ True|False ]
                (default: False)
    :param graph: Generate graph [ True|False ]
                  (default: False)
    :param trace: Generate trace [ True|False|"scorep"|"arm-map"|"arm-ddt" ]
                  (default: False)
    :param monitor: Monitor refresh rate
                    (default: None)
    :param project_xml: Project xml file path
                        (default: None)
    :param resources_xml: Resources xml file path
                          (default: None)
    :param summary: Execution summary [ True | False ]
                    (default: False)
    :param task_execution: Task execution
                           (default: "compss")
    :param storage_impl: Storage implementation path
                         (default: None)
    :param storage_conf: Storage configuration file path
                         (default: None)
    :param streaming_backend: Streaming backend
                              (default: None)
    :param streaming_master_name: Streaming master name
                                  (default: None)
    :param streaming_master_port: Streaming master port
                                  (default: None)
    :param task_count: Task count
                       (default: 50)
    :param app_name: Application name
                     default: INTERACTIVE_FILE_NAME)
    :param uuid: UUId
                 (default: None)
    :param base_log_dir: Base logging directory
                         (default: None)
    :param specific_log_dir: Specific logging directory
                             (default: None)
    :param extrae_cfg: Extrae configuration file path
                       (default: None)
    :param comm: Communication library
                 (default: NIO)
    :param conn: Connector
                 (default: DefaultSSHConnector)
    :param master_name: Master Name
                        (default: "")
    :param master_port: Master port
                        (default: "")
    :param scheduler: Scheduler (see runcompss)
                      (default: es.bsc.compss.scheduler.loadbalancing.LoadBalancingScheduler)  # noqa: E501
    :param jvm_workers: Java VM parameters
                        (default: "-Xms1024m,-Xmx1024m,-Xmn400m")
    :param cpu_affinity: CPU Core affinity
                         (default: "automatic")
    :param gpu_affinity: GPU affinity
                         (default: "automatic")
    :param fpga_affinity: FPGA affinity
                          (default: "automatic")
    :param fpga_reprogram: FPGA repogram command
                           (default: "")
    :param profile_input: Input profile
                          (default: "")
    :param profile_output: Output profile
                           (default: "")
    :param scheduler_config: Scheduler configuration
                             (default: "")
    :param external_adaptation: External adaptation [ True|False ]
                                (default: False)
    :param propagate_virtual_environment: Propagate virtual environment [ True|False ]  # noqa: E501
                                          (default: False)
    :param mpi_worker: Use the MPI worker [ True|False ]
                       (default: False)
    :param worker_cache: Use the worker cache [ True | int(size) | False]
                         (default: False)
    :param shutdown_in_node_failure: Shutdown in node failure [ True | False]
                                     (default: False)
    :param io_executors: <Integer> Number of IO executors
    :param env_script: <String> Environment script to be sourced in workers
    :param reuse_on_block: Reuse on block [ True | False]
                           (default: True)
    :param nested_enabled: Nested enabled [ True | False]
                           (default: True)
    :param tracing_task_dependencies: Include task dependencies in trace
                                      [ True | False] (default: False)
    :param trace_label: <String> Add trace label
    :param extrae_cfg_python: <String> Extrae configuration file for the
                              workers
    :param wcl: <Integer> Wall clock limit. Stops the runtime if reached.
                0 means forever.
    :param cache_profiler: Use the cache profiler [ True | False]
                         (default: False)
    :param verbose: Verbose mode [ True|False ]
                    (default: False)
    :return: None
    """
    # Export global variables
    global GRAPHING

    if context.in_pycompss():
        print("The runtime is already running")
        return None

    GRAPHING = graph
    __export_globals__()

    interactive_helpers.DEBUG = debug
    if debug:
        log_level = "debug"

    __show_flower__()

    # Let the Python binding know we are at master
    context.set_pycompss_context(context.MASTER)
    # Then we can import the appropriate start and stop functions from the API
    from pycompss.api.api import compss_start

    ##############################################################
    # INITIALIZATION
    ##############################################################

    # Initial dictionary with the user defined parameters
    all_vars = parameters_to_dict(
        log_level, debug, o_c, graph, trace, monitor, project_xml,
        resources_xml, summary, task_execution, storage_impl, storage_conf,
        streaming_backend, streaming_master_name, streaming_master_port,
        task_count, app_name, uuid, base_log_dir, specific_log_dir, extrae_cfg,
        comm, conn, master_name, master_port, scheduler, jvm_workers,
        cpu_affinity, gpu_affinity, fpga_affinity, fpga_reprogram,
        profile_input, profile_output, scheduler_config, external_adaptation,
        propagate_virtual_environment, mpi_worker, worker_cache,
        shutdown_in_node_failure, io_executors, env_script, reuse_on_block,
        nested_enabled, tracing_task_dependencies, trace_label,
        extrae_cfg_python, wcl, cache_profiler)
    # Save all vars in global current flags so that events.py can restart
    # the notebook with the same flags
    export_current_flags(all_vars)

    # Check the provided flags
    flags, issues = check_flags(all_vars)
    if not flags:
        print_flag_issues(issues)
        return None

    # Prepare the environment
    env_vars = prepare_environment(True, o_c, storage_impl, "undefined", debug,
                                   trace, mpi_worker)
    all_vars.update(env_vars)

    # Update the log level and graph values if monitoring is enabled
    monitoring_vars = prepare_loglevel_graph_for_monitoring(
        monitor, graph, debug, log_level)
    all_vars.update(monitoring_vars)

    # Check if running in supercomputer and update the variables accordingly
    # with the defined in the launcher and exported in environment variables.
    if RUNNING_IN_SUPERCOMPUTER:
        updated_vars = updated_variables_in_sc()
        if verbose:
            print("- Overridden project xml with: %s" %
                  updated_vars["project_xml"])
            print("- Overridden resources xml with: %s" %
                  updated_vars["resources_xml"])
            print("- Overridden master name with: %s" %
                  updated_vars["master_name"])
            print("- Overridden master port with: %s" %
                  updated_vars["master_port"])
            print("- Overridden uuid with: %s" % updated_vars["uuid"])
            print("- Overridden base log dir with: %s" %
                  updated_vars["base_log_dir"])
            print("- Overridden specific log dir with: %s" %
                  updated_vars["specific_log_dir"])
            print("- Overridden storage conf with: %s" %
                  updated_vars["storage_conf"])
            print("- Overridden log level with: %s" %
                  str(updated_vars["log_level"]))
            print("- Overridden debug with: %s" % str(updated_vars["debug"]))
            print("- Overridden trace with: %s" % str(updated_vars["trace"]))
        all_vars.update(updated_vars)

    # Update the tracing environment if set and set the appropriate trace
    # integer value
    tracing_vars = prepare_tracing_environment(all_vars["trace"],
                                               all_vars["extrae_lib"],
                                               all_vars["ld_library_path"])
    all_vars["trace"], all_vars["ld_library_path"] = tracing_vars

    # Update the infrastructure variables if necessary
    inf_vars = check_infrastructure_variables(all_vars["project_xml"],
                                              all_vars["resources_xml"],
                                              all_vars["compss_home"],
                                              all_vars["app_name"],
                                              all_vars["file_name"],
                                              all_vars["external_adaptation"])
    all_vars.update(inf_vars)

    # With all this information, create the configuration file for the
    # runtime start
    create_init_config_file(**all_vars)

    # Start the event manager (ipython hooks)
    ipython = globals()["__builtins__"]["get_ipython"]()
    setup_event_manager(ipython)

    ##############################################################
    # RUNTIME START
    ##############################################################

    print("* - Starting COMPSs runtime...                         *")
    sys.stdout.flush()  # Force flush
    compss_start(log_level, all_vars["trace"], True)

    global LOG_PATH
    LOG_PATH = get_log_path()
    set_temporary_directory(LOG_PATH)
    print("* - Log path : " + LOG_PATH)

    # Setup logging
    binding_log_path = get_log_path()
    log_path = os.path.join(all_vars["compss_home"], "Bindings", "python",
                            str(all_vars["major_version"]), "log")
    set_temporary_directory(binding_log_path)
    logging_cfg_file = get_logging_cfg_file(log_level)
    init_logging(os.path.join(log_path, logging_cfg_file), binding_log_path)
    logger = logging.getLogger("pycompss.runtime.launch")

    __print_setup__(verbose, all_vars)

    logger.debug("--- START ---")
    logger.debug("PyCOMPSs Log path: %s" % LOG_PATH)

    logger.debug("Starting storage")
    global PERSISTENT_STORAGE
    PERSISTENT_STORAGE = master_init_storage(all_vars["storage_conf"], logger)

    logger.debug("Starting streaming")
    global STREAMING
    STREAMING = init_streaming(all_vars["streaming_backend"],
                               all_vars["streaming_master_name"],
                               all_vars["streaming_master_port"])

    # Start monitoring the stdout and stderr
    STDW.start_watching()

    # MAIN EXECUTION
    # let the user write an interactive application
    print("* - PyCOMPSs Runtime started... Have fun!              *")
    print(LINE_SEPARATOR)

    # Emit the application start event (the 0 is in the stop function)
    emit_manual_event(APPLICATION_RUNNING_EVENT)
示例#9
0
def launch_pycompss_application(
        app,
        func,
        log_level='off',
        o_c=False,
        debug=False,
        graph=False,
        trace=False,
        monitor=None,
        project_xml=None,
        resources_xml=None,
        summary=False,
        task_execution='compss',
        storage_impl=None,
        storage_conf=None,
        task_count=50,
        app_name=None,
        uuid=None,
        base_log_dir=None,
        specific_log_dir=None,
        extrae_cfg=None,
        comm='NIO',
        conn='es.bsc.compss.connectors.DefaultSSHConnector',
        master_name='',
        master_port='',
        scheduler='es.bsc.compss.scheduler.loadBalancingScheduler.LoadBalancingScheduler',
        jvm_workers='-Xms1024m,-Xmx1024m,-Xmn400m',
        obj_conv=False,
        cpu_affinity='automatic',
        gpu_affinity='automatic',
        fpga_affinity='automatic',
        fpga_reprogram='',
        profile_input='',
        profile_output='',
        scheduler_config='',
        external_adaptation=False,
        python_propagate_virtual_environment=True,
        *args,
        **kwargs):
    """
    Launch pycompss application.

    :param app: Application path
    :param func: Function
    :param args: Arguments
    :param kwargs: Keyword arguments
    :param log_level: Logging level [ 'on' | 'off'] (default: 'off')
    :param o_c: Objects to string conversion [ True | False ] (default: False)
    :param debug: Debug mode [ True | False ] (default: False)
    :param graph: Generate graph [ True | False ] (default: False)
    :param trace: Generate trace [ True | False ] (default: False)
    :param monitor: Monitor refresh rate (default: None)
    :param project_xml: Project xml file path
    :param resources_xml: Resources xml file path
    :param summary: Execution summary [ True | False ] (default: False)
    :param task_execution: Task execution (default: 'compss')
    :param storage_impl: Storage implementation path
    :param storage_conf: Storage configuration file path
    :param task_count: Task count (default: 50)
    :param app_name: Application name (default: Interactive_date)
    :param uuid: UUId
    :param base_log_dir: Base logging directory
    :param specific_log_dir: Specific logging directory
    :param extrae_cfg: Extrae configuration file path
    :param comm: Communication library (default: NIO)
    :param conn: Connector (default: DefaultSSHConnector)
    :param master_name: Master Name (default: '')
    :param master_port: Master port (default: '')
    :param scheduler: Scheduler (default: LoadBalancingScheduler)
    :param jvm_workers: Java VM parameters (default: '-Xms1024m,-Xmx1024m,-Xmn400m')
    :param cpu_affinity: CPU Core affinity (default: 'automatic')
    :param gpu_affinity: GPU Core affinity (default: 'automatic')
    :param fpga_affinity: FPA Core affinity (default: 'automatic')
    :param fpga_reprogram: FPGA repogram command (default: '')
    :param profile_input: Input profile  (default: '')
    :param profile_output: Output profile  (default: '')
    :param scheduler_config: Scheduler configuration  (default: '')
    :param external_adaptation: External adaptation [ True | False ] (default: False)
    :param python_propagate_virtual_environment: Propagate virtual environment [ True | False ] (default: False)
    :return: Execution result
    """

    global app_path

    # Let the Python binding know we are at master
    set_pycompss_context('MASTER')

    launch_path = os.path.dirname(os.path.abspath(__file__))
    # compss_home = launch_path without the last 4 folders:
    # (Bindings/python/pycompss/runtime)
    compss_home = os.path.sep.join(launch_path.split(os.path.sep)[:-4])

    # Grab the existing PYTHONPATH and CLASSPATH values
    pythonpath = os.environ['PYTHONPATH']
    classpath = os.environ['CLASSPATH']
    ld_library_path = os.environ['LD_LIBRARY_PATH']

    # Enable/Disable object to string conversion
    binding.object_conversion = obj_conv

    # Get the filename and its path.
    file_name = os.path.splitext(os.path.basename(app))[0]
    cp = os.path.dirname(app)

    # Set storage classpath
    if storage_impl:
        if storage_impl == 'redis':
            cp = cp + ':' + compss_home + '/Tools/storage/redis/compss-redisPSCO.jar'
        else:
            cp = cp + ':' + storage_impl

    if RUNNING_IN_SUPERCOMPUTER:
        # Since the deployment in supercomputers is done through the use of enqueue_compss
        # and consequently launch_compss - the project and resources xmls are already created
        project_xml, resources_xml = get_xmls()
        # It also exported some environment variables that we need here
        master_name = get_master_node()
        master_port = get_master_port()
        uuid = get_uuid()
        base_log_dir = get_base_log_dir()
        specific_log_dir = get_specific_log_dir()
        storage_conf = get_storage_conf()
        # Override debug considering the parameter defined in pycompss_interactive_sc script
        # and exported by launch_compss
        log_level = get_log_level()
        if log_level == 'off':
            debug = False
        else:
            debug = True
        # Override tracing considering the parameter defined in pycompss_interactive_sc script
        # and exported by launch_compss
        if get_tracing():
            trace = 1
        else:
            trace = 0

    # Build a dictionary with all variables needed for initializing the runtime.
    config = dict()
    config['compss_home'] = compss_home
    config['debug'] = debug
    if project_xml is None:
        project_xml = 'Runtime/configuration/xml/projects/default_project.xml'
        config['project_xml'] = compss_home + os.path.sep + project_xml
    else:
        config['project_xml'] = project_xml
    if resources_xml is None:
        resources_xml = 'Runtime/configuration/xml/resources/default_resources.xml'
        config['resources_xml'] = compss_home + os.path.sep + resources_xml
    else:
        config['resources_xml'] = resources_xml
    config['summary'] = summary
    config['task_execution'] = task_execution
    config['storage_conf'] = storage_conf
    config['task_count'] = task_count
    if app_name is None:
        config['app_name'] = file_name
    else:
        config['app_name'] = app_name
    config['uuid'] = uuid
    config['base_log_dir'] = base_log_dir
    config['specific_log_dir'] = specific_log_dir
    config['graph'] = graph
    config['monitor'] = monitor
    config['trace'] = trace
    config['extrae_cfg'] = extrae_cfg
    config['comm'] = comm
    config['conn'] = conn
    config['master_name'] = master_name
    config['master_port'] = master_port
    config['scheduler'] = scheduler
    config['cp'] = cp
    config['classpath'] = classpath
    config['ld_library_path'] = ld_library_path
    config['jvm_workers'] = jvm_workers
    config['pythonpath'] = pythonpath
    config['cpu_affinity'] = cpu_affinity
    config['gpu_affinity'] = gpu_affinity
    config['fpga_affinity'] = fpga_affinity
    config['fpga_reprogram'] = fpga_reprogram
    config['profile_input'] = profile_input
    config['profile_output'] = profile_output
    config['scheduler_config'] = scheduler_config
    if external_adaptation:
        config['external_adaptation'] = 'true'
    else:
        config['external_adaptation'] = 'false'
    major_version = str(sys.version_info[0])
    config['python_interpreter'] = 'python' + major_version
    config['python_version'] = major_version
    if 'VIRTUAL_ENV' in os.environ:
        # Running within a virtual environment
        python_virtual_environment = os.environ['VIRTUAL_ENV']
    else:
        python_virtual_environment = 'null'
    config['python_virtual_environment'] = python_virtual_environment
    config[
        'python_propagate_virtual_environment'] = python_propagate_virtual_environment

    initialize_compss(config)

    # Runtime start
    compss_start()

    # Configure logging
    app_path = app
    log_path = get_log_path()
    # Logging setup
    if debug or log_level == "debug":
        json_path = '/Bindings/python/' + major_version + '/log/logging.json.debug'
        init_logging(compss_home + json_path, log_path)
    elif log_level == "info":
        json_path = '/Bindings/python/' + major_version + '/log/logging.json.off'
        init_logging(compss_home + json_path, log_path)
    elif log_level == "off":
        json_path = '/Bindings/python/' + major_version + '/log/logging.json.off'
        init_logging(compss_home + json_path, log_path)
    else:
        # Default
        json_path = '/Bindings/python/' + str(
            major_version) + '/log/logging.json'
        init_logging(compss_home + json_path, log_path)
    logger = logging.getLogger("pycompss.runtime.launch")

    logger.debug('--- START ---')
    logger.debug('PyCOMPSs Log path: %s' % log_path)
    saved_argv = sys.argv
    sys.argv = args
    # Execution:
    if func is None or func == '__main__':
        execfile(app)
        result = None
    else:
        import imp
        imported_module = imp.load_source(file_name, app)
        method_to_call = getattr(imported_module, func)
        result = method_to_call(*args, **kwargs)
    # Recover the system arguments
    sys.argv = saved_argv
    logger.debug('--- END ---')

    compss_stop()

    return result
示例#10
0
def start(
        log_level='off',
        o_c=False,
        debug=False,
        graph=False,
        trace=False,
        monitor=None,
        project_xml=None,
        resources_xml=None,
        summary=False,
        task_execution='compss',
        storage_impl=None,
        storage_conf=None,
        task_count=50,
        app_name='Interactive',
        uuid=None,
        base_log_dir=None,
        specific_log_dir=None,
        extrae_cfg=None,
        comm='NIO',
        conn='es.bsc.compss.connectors.DefaultSSHConnector',
        master_name='',
        master_port='',
        scheduler='es.bsc.compss.scheduler.loadBalancingScheduler.LoadBalancingScheduler',
        jvm_workers='-Xms1024m,-Xmx1024m,-Xmn400m',
        cpu_affinity='automatic',
        gpu_affinity='automatic',
        fpga_affinity='automatic',
        fpga_reprogram='',
        profile_input='',
        profile_output='',
        scheduler_config='',
        external_adaptation=False,
        propagate_virtual_environment=False,
        verbose=False):
    """
    Start the runtime in interactive mode.

    :param log_level: Logging level [ 'on' | 'off'] (default: 'off')
    :param o_c: Objects to string conversion [ True | False ] (default: False)
    :param debug: Debug mode [ True | False ] (default: False)
    :param graph: Generate graph [ True | False ] (default: False)
    :param trace: Generate trace [ True | False ] (default: False)
    :param monitor: Monitor refresh rate (default: None)
    :param project_xml: Project xml file path
    :param resources_xml: Resources xml file path
    :param summary: Execution summary [ True | False ] (default: False)
    :param task_execution: Task execution (default: 'compss')
    :param storage_impl: Storage implementation path
    :param storage_conf: Storage configuration file path
    :param task_count: Task count (default: 50)
    :param app_name: Application name (default: Interactive_date)
    :param uuid: UUId
    :param base_log_dir: Base logging directory
    :param specific_log_dir: Specific logging directory
    :param extrae_cfg: Extrae configuration file path
    :param comm: Communication library (default: NIO)
    :param conn: Connector (default: DefaultSSHConnector)
    :param master_name: Master Name (default: '')
    :param master_port: Master port (default: '')
    :param scheduler: Scheduler (default: LoadBalancingScheduler)
    :param jvm_workers: Java VM parameters (default: '-Xms1024m,-Xmx1024m,-Xmn400m')
    :param cpu_affinity: CPU Core affinity (default: 'automatic')
    :param gpu_affinity: GPU Core affinity (default: 'automatic')
    :param fpga_affinity: FPA Core affinity (default: 'automatic')
    :param fpga_reprogram: FPGA repogram command (default: '')
    :param profile_input: Input profile  (default: '')
    :param profile_output: Output profile  (default: '')
    :param scheduler_config: Scheduler configuration  (default: '')
    :param external_adaptation: External adaptation [ True | False ] (default: False)
    :param propagate_virtual_environment: Propagate virtual environment [ True | False ] (default: False)
    :param verbose: Verbose mode [ True | False ] (default: False)
    :return: None
    """

    # Let the Python binding know we are at master
    set_pycompss_context('MASTER')

    # Prepare the environment
    launch_path = os.path.dirname(os.path.realpath(__file__))
    # compss_home = launch_path without the last 4 folders:
    # Bindings/python/version/pycompss
    compss_home = os.path.sep.join(launch_path.split(os.path.sep)[:-4])
    os.environ['COMPSS_HOME'] = compss_home

    # Get environment variables
    cp = os.getcwd() + '/'
    pythonpath = os.environ['PYTHONPATH']
    classpath = os.environ['CLASSPATH']
    ld_library_path = os.environ['LD_LIBRARY_PATH']

    # Set storage classpath
    if storage_impl:
        if storage_impl == 'redis':
            cp = cp + ':' + compss_home + '/Tools/storage/redis/compss-redisPSCO.jar'
        else:
            cp = cp + ':' + storage_impl

    # Set extrae dependencies
    if not "EXTRAE_HOME" in os.environ:
        # It can be defined by the user or by launch_compss when running in Supercomputer
        extrae_home = compss_home + '/Dependencies/extrae'
        os.environ['EXTRAE_HOME'] = extrae_home
    else:
        extrae_home = os.environ['EXTRAE_HOME']
    extrae_lib = extrae_home + '/lib'

    # Include extrae into ld_library_path
    ld_library_path = extrae_lib + ':' + ld_library_path
    os.environ['LD_LIBRARY_PATH'] = ld_library_path

    if monitor is not None:
        # Enable the graph if the monitoring is enabled
        graph = True

    # Export global variables
    global graphing
    graphing = graph

    __export_globals__()

    print("******************************************************")
    print("*************** PyCOMPSs Interactive *****************")
    print("******************************************************")
    print("*          .-~~-.--.            ____        ____     *")
    print("*         :         )          |___ \      |___ \    *")
    print("*   .~ ~ -.\       /.- ~~ .      __) |       __) |   *")
    print("*   >       `.   .'       <     / __/   _   ___) |   *")
    print("*  (         .- -.         )   |_____| |_| |____/    *")
    print("*   `- -.-~  `- -'  ~-.- -'                          *")
    print("*     (        :        )           _ _ .-:          *")
    print("*      ~--.    :    .--~        .-~  .-~  }          *")
    print("*          ~-.-^-.-~ \_      .~  .-~   .~            *")
    print("*                   \ \ '     \ '_ _ -~              *")
    print("*                    \`.\`.    //                    *")
    print("*           . - ~ ~-.__\`.\`-.//                     *")
    print("*       .-~   . - ~  }~ ~ ~-.~-.                     *")
    print("*     .' .-~      .-~       :/~-.~-./:               *")
    print("*    /_~_ _ . - ~                 ~-.~-._            *")
    print("*                                     ~-.<           *")
    print("******************************************************")

    ##############################################################
    # INITIALIZATION
    ##############################################################

    if RUNNING_IN_SUPERCOMPUTER:
        # Since the deployment in supercomputers is done through the use of enqueue_compss
        # and consequently launch_compss - the project and resources xmls are already created
        project_xml, resources_xml = get_xmls()
        # It also exported some environment variables that we need here
        master_name = get_master_node()
        master_port = get_master_port()
        uuid = get_uuid()
        base_log_dir = get_base_log_dir()
        specific_log_dir = get_specific_log_dir()
        storage_conf = get_storage_conf()
        # Override debug considering the parameter defined in pycompss_interactive_sc script
        # and exported by launch_compss
        log_level = get_log_level()
        if log_level == 'off':
            debug = False
        else:
            debug = True
        # Override tracing considering the parameter defined in pycompss_interactive_sc script
        # and exported by launch_compss
        trace = get_tracing()
        if verbose:
            print("- Overridden project xml with: " + project_xml)
            print("- Overridden resources xml with: " + resources_xml)
            print("- Overridden master name with: " + master_name)
            print("- Overridden master port with: " + master_port)
            print("- Overridden uuid with: " + uuid)
            print("- Overridden base log dir with: " + base_log_dir)
            print("- Overridden specific log dir with: " + specific_log_dir)
            print("- Overridden storage conf with: " + storage_conf)
            print("- Overridden log level with: " + str(log_level))
            print("- Overridden debug with: " + str(debug))
            print("- Overridden trace with: " + str(trace))

    if debug:
        # Add environment variable to get binding-commons debug information
        os.environ['COMPSS_BINDINGS_DEBUG'] = '1'

    if trace is False:
        trace = 0
    elif trace == 'basic' or trace is True:
        trace = 1
        os.environ['LD_PRELOAD'] = extrae_lib + '/libpttrace.so'
    elif trace == 'advanced':
        trace = 2
        os.environ['LD_PRELOAD'] = extrae_lib + '/libpttrace.so'
    else:
        print(
            "ERROR: Wrong tracing parameter ( [ True | basic ] | advanced | False)"
        )
        return -1

    # Build a dictionary with all variables needed for initializing the runtime
    config = dict()
    config['compss_home'] = compss_home
    config['debug'] = debug
    if project_xml is None:
        project_path = 'Runtime/configuration/xml/projects/default_project.xml'
        config['project_xml'] = compss_home + os.path.sep + project_path
    else:
        config['project_xml'] = project_xml
    if resources_xml is None:
        resources_path = 'Runtime/configuration/xml/resources/default_resources.xml'
        config['resources_xml'] = compss_home + os.path.sep + resources_path
    else:
        config['resources_xml'] = resources_xml
    config['summary'] = summary
    config['task_execution'] = task_execution
    config['storage_conf'] = storage_conf
    config['task_count'] = task_count
    if app_name is None:
        config['app_name'] = 'Interactive'
    else:
        config['app_name'] = app_name
    config['uuid'] = uuid
    config['base_log_dir'] = base_log_dir
    config['specific_log_dir'] = specific_log_dir
    config['graph'] = graph
    config['monitor'] = monitor
    config['trace'] = trace
    config['extrae_cfg'] = extrae_cfg
    config['comm'] = comm
    config['conn'] = conn
    config['master_name'] = master_name
    config['master_port'] = master_port
    config['scheduler'] = scheduler
    config['cp'] = cp
    config['pythonpath'] = pythonpath
    config['classpath'] = classpath
    config['ld_library_path'] = ld_library_path
    config['jvm_workers'] = jvm_workers
    config['cpu_affinity'] = cpu_affinity
    config['gpu_affinity'] = gpu_affinity
    config['fpga_affinity'] = fpga_affinity
    config['fpga_reprogram'] = fpga_reprogram
    config['profile_input'] = profile_input
    config['profile_output'] = profile_output
    config['scheduler_config'] = scheduler_config
    if external_adaptation:
        config['external_adaptation'] = 'true'
    else:
        config['external_adaptation'] = 'false'

    major_version = sys.version_info[0]
    python_interpreter = 'python' + str(major_version)
    config['python_interpreter'] = python_interpreter
    config['python_version'] = str(major_version)

    if 'VIRTUAL_ENV' in os.environ:
        # Running within a virtual environment
        python_virtual_environment = os.environ['VIRTUAL_ENV']
    else:
        python_virtual_environment = 'null'
    config['python_virtual_environment'] = python_virtual_environment
    config[
        'python_propagate_virtual_environment'] = propagate_virtual_environment

    initialize_compss(config)

    ##############################################################
    # RUNTIME START
    ##############################################################

    print("* - Starting COMPSs runtime...                       *")
    sys.stdout.flush()  # Force flush
    compss_start()

    if o_c is True:
        # set cross-module variable
        binding.object_conversion = True
    else:
        # set cross-module variable
        binding.object_conversion = False

    # Remove launch.py, log_level and object_conversion from sys.argv,
    # It will be inherited by the app through execfile
    # sys.argv = sys.argv[3:]
    # Get application execution path
    # app_path = sys.argv[0]  # not needed in interactive mode

    global log_path
    log_path = get_log_path()
    binding.temp_dir = mkdtemp(prefix='pycompss', dir=log_path + '/tmpFiles/')
    print("* - Log path : " + log_path)

    # Logging setup - messages before this step are ignored (need log_path to configure the logger).
    if debug or log_level == "debug":
        json_path = '/Bindings/python/' + str(
            major_version) + '/log/logging.json.debug'
        init_logging(os.getenv('COMPSS_HOME') + json_path, log_path)
    elif log_level == "info":
        json_path = '/Bindings/python/' + str(
            major_version) + '/log/logging.json.off'
        init_logging(os.getenv('COMPSS_HOME') + json_path, log_path)
    elif log_level == "off":
        json_path = '/Bindings/python/' + str(
            major_version) + '/log/logging.json.off'
        init_logging(os.getenv('COMPSS_HOME') + json_path, log_path)
    else:
        # Default
        json_path = '/Bindings/python/' + str(
            major_version) + '/log/logging.json'
        init_logging(os.getenv('COMPSS_HOME') + json_path, log_path)
    logger = logging.getLogger("pycompss.runtime.launch")

    __print_setup__(verbose, log_level, o_c, debug, graph, trace, monitor,
                    project_xml, resources_xml, summary, task_execution,
                    storage_conf, pythonpath, classpath, ld_library_path,
                    task_count, app_name, uuid, base_log_dir, specific_log_dir,
                    extrae_cfg, comm, conn, master_name, master_port,
                    scheduler, jvm_workers, cpu_affinity, gpu_affinity,
                    fpga_affinity, fpga_reprogram, profile_input,
                    profile_output, scheduler_config, external_adaptation,
                    python_interpreter, major_version,
                    python_virtual_environment, propagate_virtual_environment)

    logger.debug("--- START ---")
    logger.debug("PyCOMPSs Log path: %s" % log_path)
    if storage_conf is not None:
        logger.debug("Storage configuration file: %s" % storage_conf)
        from storage.api import init as init_storage
        init_storage(config_file_path=storage_conf)
        global persistent_storage
        persistent_storage = True

    # MAIN EXECUTION
    # let the user write an interactive application
    print("* - PyCOMPSs Runtime started... Have fun!            *")
    print("******************************************************")
示例#11
0
def launch_pycompss_application(
        app,  # type: str
        func,  # type: typing.Optional[str]
        log_level="off",  # type: str
        o_c=False,  # type: bool
        debug=False,  # type: bool
        graph=False,  # type: bool
        trace=False,  # type: bool
        monitor=-1,  # type: int
        project_xml="",  # type: str
        resources_xml="",  # type: str
        summary=False,  # type: bool
        task_execution="compss",  # type: str
        storage_impl="",  # type: str
        storage_conf="",  # type: str
        streaming_backend="",  # type: str
        streaming_master_name="",  # type: str
        streaming_master_port="",  # type: str
        task_count=50,  # type: int
        app_name="",  # type: str
        uuid="",  # type: str
        base_log_dir="",  # type: str
        specific_log_dir="",  # type: str
        extrae_cfg="",  # type: str
        comm="NIO",  # type: str
        conn=DEFAULT_CONN,  # type: str
        master_name="",  # type: str
        master_port="",  # type: str
        scheduler=DEFAULT_SCHED,  # type: str
        jvm_workers=DEFAULT_JVM_WORKERS,  # type: str
        cpu_affinity="automatic",  # type: str
        gpu_affinity="automatic",  # type: str
        fpga_affinity="automatic",  # type: str
        fpga_reprogram="",  # type: str
        profile_input="",  # type: str
        profile_output="",  # type: str
        scheduler_config="",  # type: str
        external_adaptation=False,  # type: bool
        propagate_virtual_environment=True,  # type: bool
        mpi_worker=False,  # type: bool
        worker_cache=False,  # type: typing.Union[bool, str]
        shutdown_in_node_failure=False,  # type: bool
        io_executors=0,  # type: int
        env_script="",  # type: str
        reuse_on_block=True,  # type: bool
        nested_enabled=False,  # type: bool
        tracing_task_dependencies=False,  # type: bool
        trace_label="",  # type: str
        extrae_cfg_python="",  # type: str
        wcl=0,  # type: int
        cache_profiler=False,  # type: bool
        *args,
        **kwargs):  # NOSONAR
    # type: (...) -> typing.Any
    """ Launch PyCOMPSs application from function.

    :param app: Application path
    :param func: Function
    :param log_level: Logging level [ "trace"|"debug"|"info"|"api"|"off" ]
                      (default: "off")
    :param o_c: Objects to string conversion [ True | False ] (default: False)
    :param debug: Debug mode [ True | False ] (default: False)
                  (overrides log_level)
    :param graph: Generate graph [ True | False ] (default: False)
    :param trace: Generate trace
                  [ True | False | "scorep" | "arm-map" | "arm-ddt"]
                  (default: False)
    :param monitor: Monitor refresh rate (default: None)
    :param project_xml: Project xml file path
    :param resources_xml: Resources xml file path
    :param summary: Execution summary [ True | False ] (default: False)
    :param task_execution: Task execution (default: "compss")
    :param storage_impl: Storage implementation path
    :param storage_conf: Storage configuration file path
    :param streaming_backend: Streaming backend (default: None)
    :param streaming_master_name: Streaming master name (default: None)
    :param streaming_master_port: Streaming master port (default: None)
    :param task_count: Task count (default: 50)
    :param app_name: Application name (default: Interactive_date)
    :param uuid: UUId
    :param base_log_dir: Base logging directory
    :param specific_log_dir: Specific logging directory
    :param extrae_cfg: Extrae configuration file path
    :param comm: Communication library (default: NIO)
    :param conn: Connector (default: DefaultSSHConnector)
    :param master_name: Master Name (default: "")
    :param master_port: Master port (default: "")
    :param scheduler: Scheduler (default:
                  es.bsc.compss.scheduler.loadbalancing.LoadBalancingScheduler)
    :param jvm_workers: Java VM parameters
                        (default: "-Xms1024m,-Xmx1024m,-Xmn400m")
    :param cpu_affinity: CPU Core affinity (default: "automatic")
    :param gpu_affinity: GPU Core affinity (default: "automatic")
    :param fpga_affinity: FPA Core affinity (default: "automatic")
    :param fpga_reprogram: FPGA reprogram command (default: "")
    :param profile_input: Input profile  (default: "")
    :param profile_output: Output profile  (default: "")
    :param scheduler_config: Scheduler configuration  (default: "")
    :param external_adaptation: External adaptation [ True | False ]
                                (default: False)
    :param propagate_virtual_environment: Propagate virtual environment
                                          [ True | False ] (default: False)
    :param mpi_worker: Use the MPI worker [ True | False ] (default: False)
    :param worker_cache: Use the worker cache [ True | int(size) | False]
                         (default: False)
    :param shutdown_in_node_failure: Shutdown in node failure [ True | False]
                                     (default: False)
    :param io_executors: <Integer> Number of IO executors
    :param env_script: <String> Environment script to be sourced in workers
    :param reuse_on_block: Reuse on block [ True | False] (default: True)
    :param nested_enabled: Nested enabled [ True | False] (default: False)
    :param tracing_task_dependencies: Include task dependencies in trace
                                      [ True | False] (default: False)
    :param trace_label: <String> Add trace label
    :param extrae_cfg_python: <String> Extrae configuration file for the
                              workers
    :param wcl: <Integer> Wallclock limit. Stops the runtime if reached.
                0 means forever.
    :param cache_profiler: Use the cache profiler [ True | False]
                         (default: False)
    :param args: Positional arguments
    :param kwargs: Named arguments
    :return: Execution result
    """
    # Check that COMPSs is available
    if "COMPSS_HOME" not in os.environ:
        # Do not allow to continue if COMPSS_HOME is not defined
        raise PyCOMPSsException(
            "ERROR: COMPSS_HOME is not defined in the environment")

    # Let the Python binding know we are at master
    context.set_pycompss_context(context.MASTER)
    # Then we can import the appropriate start and stop functions from the API
    from pycompss.api.api import compss_start, compss_stop

    ##############################################################
    # INITIALIZATION
    ##############################################################

    if debug:
        log_level = "debug"

    # Initial dictionary with the user defined parameters
    all_vars = parameters_to_dict(
        log_level, debug, o_c, graph, trace, monitor, project_xml,
        resources_xml, summary, task_execution, storage_impl, storage_conf,
        streaming_backend, streaming_master_name, streaming_master_port,
        task_count, app_name, uuid, base_log_dir, specific_log_dir, extrae_cfg,
        comm, conn, master_name, master_port, scheduler, jvm_workers,
        cpu_affinity, gpu_affinity, fpga_affinity, fpga_reprogram,
        profile_input, profile_output, scheduler_config, external_adaptation,
        propagate_virtual_environment, mpi_worker, worker_cache,
        shutdown_in_node_failure, io_executors, env_script, reuse_on_block,
        nested_enabled, tracing_task_dependencies, trace_label,
        extrae_cfg_python, wcl, cache_profiler)
    # Save all vars in global current flags so that events.py can restart
    # the notebook with the same flags
    export_current_flags(all_vars)

    # Check the provided flags
    flags, issues = check_flags(all_vars)
    if not flags:
        print_flag_issues(issues)
        return None

    # Prepare the environment
    env_vars = prepare_environment(False, o_c, storage_impl, app, debug, trace,
                                   mpi_worker)
    all_vars.update(env_vars)

    monitoring_vars = prepare_loglevel_graph_for_monitoring(
        monitor, graph, debug, log_level)
    all_vars.update(monitoring_vars)

    if RUNNING_IN_SUPERCOMPUTER:
        updated_vars = updated_variables_in_sc()
        all_vars.update(updated_vars)

    to_update = prepare_tracing_environment(all_vars["trace"],
                                            all_vars["extrae_lib"],
                                            all_vars["ld_library_path"])
    all_vars["trace"], all_vars["ld_library_path"] = to_update

    inf_vars = check_infrastructure_variables(all_vars["project_xml"],
                                              all_vars["resources_xml"],
                                              all_vars["compss_home"],
                                              all_vars["app_name"],
                                              all_vars["file_name"],
                                              all_vars["external_adaptation"])
    all_vars.update(inf_vars)

    create_init_config_file(**all_vars)

    ##############################################################
    # RUNTIME START
    ##############################################################

    # Runtime start
    compss_start(log_level, all_vars["trace"], True)

    # Setup logging
    binding_log_path = get_log_path()
    log_path = os.path.join(all_vars["compss_home"], "Bindings", "python",
                            str(all_vars["major_version"]), "log")
    set_temporary_directory(binding_log_path)
    logging_cfg_file = get_logging_cfg_file(log_level)
    init_logging(os.path.join(log_path, logging_cfg_file), binding_log_path)
    logger = logging.getLogger("pycompss.runtime.launch")

    logger.debug("--- START ---")
    logger.debug("PyCOMPSs Log path: %s" % log_path)

    if storage_impl and storage_conf:
        logger.debug("Starting storage")
        persistent_storage = master_init_storage(all_vars["storage_conf"],
                                                 logger)
    else:
        persistent_storage = False

    logger.debug("Starting streaming")
    streaming = init_streaming(all_vars["streaming_backend"],
                               all_vars["streaming_master_name"],
                               all_vars["streaming_master_port"])

    saved_argv = sys.argv
    sys.argv = list(args)
    # Execution:
    with event_master(APPLICATION_RUNNING_EVENT):
        if func is None or func == "__main__":
            if IS_PYTHON3:
                exec(open(app).read())
            else:
                execfile(app)  # type: ignore
            result = None
        else:
            if IS_PYTHON3:
                from importlib.machinery import SourceFileLoader  # noqa
                imported_module = SourceFileLoader(
                    all_vars["file_name"], app).load_module()  # type: ignore
            else:
                import imp  # noqa
                imported_module = imp.load_source(all_vars["file_name"],
                                                  app)  # noqa
            method_to_call = getattr(imported_module, func)
            try:
                result = method_to_call(*args, **kwargs)
            except TypeError:
                result = method_to_call()
    # Recover the system arguments
    sys.argv = saved_argv

    # Stop streaming
    if streaming:
        stop_streaming()

    # Stop persistent storage
    if persistent_storage:
        master_stop_storage(logger)

    logger.debug("--- END ---")

    ##############################################################
    # RUNTIME STOP
    ##############################################################

    # Stop runtime
    compss_stop()
    clean_log_configs()

    return result
示例#12
0
def compss_main():
    # type: () -> None
    """ PyCOMPSs main function.

    General call:
    python $PYCOMPSS_HOME/pycompss/runtime/launch.py $wall_clock $log_level
           $PyObject_serialize $storage_conf $streaming_backend
           $streaming_master_name $streaming_master_port
           $fullAppPath $application_args

    :return: None
    """
    global APP_PATH
    global STREAMING
    global PERSISTENT_STORAGE
    global LOGGER
    # Let the Python binding know we are at master
    context.set_pycompss_context(context.MASTER)
    # Then we can import the appropriate start and stop functions from the API
    from pycompss.api.api import compss_start  # noqa
    from pycompss.api.api import compss_stop  # noqa
    from pycompss.api.api import compss_set_wall_clock  # noqa

    # See parse_arguments, defined above
    # In order to avoid parsing user arguments, we are going to remove user
    # args from sys.argv
    user_sys_argv = sys.argv[10:]
    sys.argv = sys.argv[:10]
    args = parse_arguments()
    # We are done, now sys.argv must contain user args only
    sys.argv = [args.app_path] + user_sys_argv

    # Get log_level
    log_level = args.log_level

    # Setup tracing
    tracing = int(args.tracing)

    # Get storage configuration at master
    storage_conf = args.storage_configuration

    # Load user imports before starting the runtime (can be avoided if
    # ENVIRONMENT_VARIABLE_LOAD -- defined in configuration.py --
    # is set to false).
    # Reason: some cases like autoparallel can require to avoid loading.
    # It is disabled if using storage (with dataClay this can not be done)
    if preload_user_code() and not use_storage(storage_conf):
        with context.loading_context():
            __load_user_module__(args.app_path, log_level)

    # Start the runtime
    compss_start(log_level, tracing, False)

    # Register @implements core elements (they can not be registered in
    # __load_user__module__).
    __register_implementation_core_elements__()

    # Get application wall clock limit
    wall_clock = int(args.wall_clock)
    if wall_clock > 0:
        compss_set_wall_clock(wall_clock)

    # Get object_conversion boolean
    set_object_conversion(args.object_conversion == "true")

    # Get application execution path
    APP_PATH = args.app_path

    # Setup logging
    binding_log_path = get_log_path()
    log_path = os.path.join(str(os.getenv("COMPSS_HOME")), "Bindings",
                            "python", str(PYTHON_VERSION), "log")
    set_temporary_directory(binding_log_path)
    logging_cfg_file = get_logging_cfg_file(log_level)
    init_logging(os.path.join(log_path, logging_cfg_file), binding_log_path)
    LOGGER = logging.getLogger("pycompss.runtime.launch")

    # Get JVM options
    # jvm_opts = os.environ["JVM_OPTIONS_FILE"]
    # from pycompss.util.jvm.parser import convert_to_dict
    # opts = convert_to_dict(jvm_opts)
    # storage_conf = opts.get("-Dcompss.storage.conf")

    exit_code = 0
    try:
        if __debug__:
            LOGGER.debug("--- START ---")
            LOGGER.debug("PyCOMPSs Log path: %s" % binding_log_path)

        # Start persistent storage
        PERSISTENT_STORAGE = master_init_storage(storage_conf, LOGGER)

        # Start STREAMING
        STREAMING = init_streaming(args.streaming_backend,
                                   args.streaming_master_name,
                                   args.streaming_master_port)

        # Show module warnings
        if __debug__:
            show_optional_module_warnings()

        # MAIN EXECUTION
        with event_master(APPLICATION_RUNNING_EVENT):
            # MAIN EXECUTION
            if IS_PYTHON3:
                with open(APP_PATH) as f:
                    exec(compile(f.read(), APP_PATH, "exec"), globals())
            else:
                execfile(APP_PATH, globals())  # type: ignore

        # End
        if __debug__:
            LOGGER.debug("--- END ---")
    except SystemExit as e:  # NOSONAR - reraising would not allow to stop the runtime gracefully.
        if e.code != 0:
            print("[ ERROR ]: User program ended with exitcode %s." % e.code)
            print("\t\tShutting down runtime...")
            exit_code = e.code
    except SerializerException:
        exit_code = 1
        # If an object that can not be serialized has been used as a parameter.
        print("[ ERROR ]: Serialization exception")
        exc_type, exc_value, exc_traceback = sys.exc_info()
        lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
        for line in lines:
            if APP_PATH in line:
                print("[ ERROR ]: In: %s", line)
        exit_code = 1
    except COMPSsException as e:
        # Any other exception occurred
        print("[ ERROR ]: A COMPSs exception occurred: " + str(e))
        traceback.print_exc()
        exit_code = 0  # COMPSs exception is not considered an error
    except Exception as e:
        # Any other exception occurred
        print("[ ERROR ]: An exception occurred: " + str(e))
        traceback.print_exc()
        exit_code = 1
    finally:
        # Stop runtime
        stop_all(exit_code)
        clean_log_configs()
示例#13
0
def launch_pycompss_application(
        app,
        func,
        log_level='off',
        o_c=False,
        debug=False,
        graph=False,
        trace=False,
        monitor=None,
        project_xml=None,
        resources_xml=None,
        summary=False,
        task_execution='compss',
        storage_impl=None,
        storage_conf=None,
        streaming_backend=None,
        streaming_master_name=None,
        streaming_master_port=None,
        task_count=50,
        app_name=None,
        uuid=None,
        base_log_dir=None,
        specific_log_dir=None,
        extrae_cfg=None,
        comm='NIO',
        conn='es.bsc.compss.connectors.DefaultSSHConnector',  # noqa
        master_name='',
        master_port='',
        scheduler='es.bsc.compss.scheduler.loadbalancing.LoadBalancingScheduler',  # noqa
        jvm_workers='-Xms1024m,-Xmx1024m,-Xmn400m',
        cpu_affinity='automatic',
        gpu_affinity='automatic',
        fpga_affinity='automatic',
        fpga_reprogram='',
        profile_input='',
        profile_output='',
        scheduler_config='',
        external_adaptation=False,
        propagate_virtual_environment=True,
        mpi_worker=False,
        *args,
        **kwargs):
    """
    Launch PyCOMPSs application from function.

    :param app: Application path
    :param func: Function
    :param args: Arguments
    :param kwargs: Keyword arguments
    :param log_level: Logging level [ 'off' | 'info'  | 'debug' ]
                      (default: 'off')
    :param o_c: Objects to string conversion [ True | False ] (default: False)
    :param debug: Debug mode [ True | False ] (default: False)
                  (overrides log_level)
    :param graph: Generate graph [ True | False ] (default: False)
    :param trace: Generate trace
                  [ True | False | 'scorep' | 'arm-map' | 'arm-ddt']
                  (default: False)
    :param monitor: Monitor refresh rate (default: None)
    :param project_xml: Project xml file path
    :param resources_xml: Resources xml file path
    :param summary: Execution summary [ True | False ] (default: False)
    :param task_execution: Task execution (default: 'compss')
    :param storage_impl: Storage implementation path
    :param storage_conf: Storage configuration file path
    :param streaming_backend: Streaming backend (default: None)
    :param streaming_master_name: Streaming master name (default: None)
    :param streaming_master_port: Streaming master port (default: None)
    :param task_count: Task count (default: 50)
    :param app_name: Application name (default: Interactive_date)
    :param uuid: UUId
    :param base_log_dir: Base logging directory
    :param specific_log_dir: Specific logging directory
    :param extrae_cfg: Extrae configuration file path
    :param comm: Communication library (default: NIO)
    :param conn: Connector (default: DefaultSSHConnector)
    :param master_name: Master Name (default: '')
    :param master_port: Master port (default: '')
    :param scheduler: Scheduler (default:
                  es.bsc.compss.scheduler.loadbalancing.LoadBalancingScheduler)
    :param jvm_workers: Java VM parameters
                        (default: '-Xms1024m,-Xmx1024m,-Xmn400m')
    :param cpu_affinity: CPU Core affinity (default: 'automatic')
    :param gpu_affinity: GPU Core affinity (default: 'automatic')
    :param fpga_affinity: FPA Core affinity (default: 'automatic')
    :param fpga_reprogram: FPGA repogram command (default: '')
    :param profile_input: Input profile  (default: '')
    :param profile_output: Output profile  (default: '')
    :param scheduler_config: Scheduler configuration  (default: '')
    :param external_adaptation: External adaptation [ True | False ]
                                (default: False)
    :param propagate_virtual_environment: Propagate virtual environment
                                          [ True | False ] (default: False)
    :param mpi_worker: Use the MPI worker [ True | False ] (default: False)
    :param args: Positional arguments
    :param kwargs: Named arguments
    :return: Execution result
    """
    # Let the Python binding know we are at master
    context.set_pycompss_context(context.MASTER)
    # Then we can import the appropriate start and stop functions from the API
    from pycompss.api.api import compss_start, compss_stop

    ##############################################################
    # INITIALIZATION
    ##############################################################

    # TODO: Check that input values are valid

    # Initial dictionary with the user defined parameters
    all_vars = {
        'log_level': log_level,
        'debug': debug,
        'o_c': o_c,
        'graph': graph,
        'trace': trace,
        'monitor': monitor,
        'project_xml': project_xml,
        'resources_xml': resources_xml,
        'summary': summary,
        'task_execution': task_execution,
        'storage_impl': storage_impl,
        'storage_conf': storage_conf,
        'streaming_backend': streaming_backend,
        'streaming_master_name': streaming_master_name,
        'streaming_master_port': streaming_master_port,
        'task_count': task_count,
        'app_name': app_name,
        'uuid': uuid,
        'base_log_dir': base_log_dir,
        'specific_log_dir': specific_log_dir,
        'extrae_cfg': extrae_cfg,
        'comm': comm,
        'conn': conn,
        'master_name': master_name,
        'master_port': master_port,
        'scheduler': scheduler,
        'jvm_workers': jvm_workers,
        'cpu_affinity': cpu_affinity,
        'gpu_affinity': gpu_affinity,
        'fpga_affinity': fpga_affinity,
        'fpga_reprogram': fpga_reprogram,
        'profile_input': profile_input,
        'profile_output': profile_output,
        'scheduler_config': scheduler_config,
        'external_adaptation': external_adaptation,
        'propagate_virtual_environment': propagate_virtual_environment,
        'mpi_worker': mpi_worker
    }

    # Prepare the environment
    env_vars = prepare_environment(True, o_c, storage_impl, app, debug, trace,
                                   mpi_worker)
    all_vars.update(env_vars)

    monitoring_vars = prepare_loglevel_graph_for_monitoring(
        monitor, graph, debug, log_level)
    all_vars.update(monitoring_vars)

    if RUNNING_IN_SUPERCOMPUTER:
        updated_vars = updated_variables_in_sc()
        all_vars.update(updated_vars)

    to_update = prepare_tracing_environment(all_vars['trace'],
                                            all_vars['extrae_lib'],
                                            all_vars['ld_library_path'])
    all_vars['trace'], all_vars['ld_library_path'] = to_update

    inf_vars = check_infrastructure_variables(all_vars['project_xml'],
                                              all_vars['resources_xml'],
                                              all_vars['compss_home'],
                                              all_vars['app_name'],
                                              all_vars['file_name'],
                                              all_vars['external_adaptation'])
    all_vars.update(inf_vars)

    create_init_config_file(**all_vars)

    ##############################################################
    # RUNTIME START
    ##############################################################

    # Runtime start
    compss_start()

    # Configure logging
    log_path = get_log_path()
    major_version = all_vars['major_version']
    compss_home = all_vars['compss_home']
    logger = setup_logger(debug, log_level, major_version, compss_home,
                          log_path)

    logger.debug('--- START ---')
    logger.debug('PyCOMPSs Log path: %s' % log_path)

    logger.debug("[LOG] Starting storage")
    persistent_storage = init_storage(all_vars['storage_conf'], logger)

    logger.debug("[LOG] Starting streaming")
    streaming = init_streaming(all_vars['streaming_backend'],
                               all_vars['streaming_master_name'],
                               all_vars['streaming_master_port'])

    saved_argv = sys.argv
    sys.argv = args
    # Execution:
    if func is None or func == '__main__':
        execfile(app)
        result = None
    else:
        import imp
        imported_module = imp.load_source(all_vars['file_name'], app)
        method_to_call = getattr(imported_module, func)
        result = method_to_call(*args, **kwargs)
    # Recover the system arguments
    sys.argv = saved_argv

    if persistent_storage:
        logger.debug("[LOG] Stopping persistent storage")
        stop_storage()

    if streaming:
        logger.debug("[LOG] Stopping streaming")
        stop_streaming()

    logger.debug('--- END ---')

    ##############################################################
    # RUNTIME STOP
    ##############################################################

    compss_stop()

    return result
示例#14
0
def test_dummy_api():
    from pycompss.api.dummy.api import compss_start
    from pycompss.api.dummy.api import compss_stop
    from pycompss.api.dummy.api import compss_file_exists
    from pycompss.api.dummy.api import compss_open
    from pycompss.api.dummy.api import compss_delete_file
    from pycompss.api.dummy.api import compss_wait_on_file
    from pycompss.api.dummy.api import compss_wait_on_directory
    from pycompss.api.dummy.api import compss_delete_object
    from pycompss.api.dummy.api import compss_barrier
    from pycompss.api.dummy.api import compss_barrier_group
    from pycompss.api.dummy.api import compss_wait_on
    from pycompss.api.dummy.api import compss_get_number_of_resources
    from pycompss.api.dummy.api import compss_request_resources
    from pycompss.api.dummy.api import compss_free_resources
    from pycompss.api.dummy.api import TaskGroup

    file_name = "simulated_file.txt"
    file_names = ["simulated_file1.txt", "simulated_file2.txt"]
    directory_name = "simulated_directory"
    directory_names = ["simulated_directory1", "simulated_directory2"]
    group_name = "simulated_group"
    obj = [1, 2, 3]
    num_resources = 1

    with open(file_name, "w") as f:
        f.write("some content")
    os.mkdir(directory_name)

    for f_name in file_names:
        with open(f_name, "w") as f:
            f.write("some content")
    for d_name in directory_names:
        os.mkdir(d_name)

    compss_start(log_level="off", interactive=False)
    compss_stop(code=0)
    compss_file_exists(file_name)
    compss_file_exists(*file_names)
    compss_open(file_name, mode="r")
    compss_delete_file(file_name)
    compss_delete_file(*file_names)
    compss_wait_on_file(file_name)
    compss_wait_on_file(*file_names)
    compss_wait_on_directory(directory_name)
    compss_wait_on_directory(*directory_names)
    compss_delete_object(obj)
    compss_delete_object(*obj)
    compss_barrier(no_more_tasks=False)
    compss_barrier_group(group_name)
    compss_wait_on(obj)
    compss_wait_on(*obj)
    compss_get_number_of_resources()
    compss_request_resources(num_resources, group_name)
    compss_free_resources(num_resources, group_name)

    with TaskGroup(group_name, implicit_barrier=True):
        # Empty task group check
        pass

    os.remove(file_name)
    os.rmdir(directory_name)

    for f_name in file_names:
        os.remove(f_name)
    for d_name in directory_names:
        os.rmdir(d_name)