def launch_pycompss_application( app, func, log_level='off', # type: str o_c=False, # type: bool debug=False, # type: bool graph=False, # type: bool trace=False, # type: bool monitor=None, # type: int project_xml=None, # type: str resources_xml=None, # type: str summary=False, # type: bool task_execution='compss', # type: str storage_impl=None, # type: str storage_conf=None, # type: str streaming_backend=None, # type: str streaming_master_name=None, # type: str streaming_master_port=None, # type: str task_count=50, # type: int app_name=None, # type: str uuid=None, # type: str base_log_dir=None, # type: str specific_log_dir=None, # type: str extrae_cfg=None, # type: str comm='NIO', # type: str conn='es.bsc.compss.connectors.DefaultSSHConnector', # noqa: E501 master_name='', # type: str master_port='', # type: str scheduler='es.bsc.compss.scheduler.loadbalancing.LoadBalancingScheduler', # noqa: E501 jvm_workers='-Xms1024m,-Xmx1024m,-Xmn400m', cpu_affinity='automatic', # type: str gpu_affinity='automatic', # type: str fpga_affinity='automatic', # type: str fpga_reprogram='', # type: str profile_input='', # type: str profile_output='', # type: str scheduler_config='', # type: str external_adaptation=False, # type: bool propagate_virtual_environment=True, # type: bool mpi_worker=False, # type: bool *args, **kwargs): # type: (...) -> None """ Launch PyCOMPSs application from function. :param app: Application path :param func: Function :param log_level: Logging level [ 'trace'|'debug'|'info'|'api'|'off' ] (default: 'off') :param o_c: Objects to string conversion [ True | False ] (default: False) :param debug: Debug mode [ True | False ] (default: False) (overrides log_level) :param graph: Generate graph [ True | False ] (default: False) :param trace: Generate trace [ True | False | 'scorep' | 'arm-map' | 'arm-ddt'] (default: False) :param monitor: Monitor refresh rate (default: None) :param project_xml: Project xml file path :param resources_xml: Resources xml file path :param summary: Execution summary [ True | False ] (default: False) :param task_execution: Task execution (default: 'compss') :param storage_impl: Storage implementation path :param storage_conf: Storage configuration file path :param streaming_backend: Streaming backend (default: None) :param streaming_master_name: Streaming master name (default: None) :param streaming_master_port: Streaming master port (default: None) :param task_count: Task count (default: 50) :param app_name: Application name (default: Interactive_date) :param uuid: UUId :param base_log_dir: Base logging directory :param specific_log_dir: Specific logging directory :param extrae_cfg: Extrae configuration file path :param comm: Communication library (default: NIO) :param conn: Connector (default: DefaultSSHConnector) :param master_name: Master Name (default: '') :param master_port: Master port (default: '') :param scheduler: Scheduler (default: es.bsc.compss.scheduler.loadbalancing.LoadBalancingScheduler) :param jvm_workers: Java VM parameters (default: '-Xms1024m,-Xmx1024m,-Xmn400m') :param cpu_affinity: CPU Core affinity (default: 'automatic') :param gpu_affinity: GPU Core affinity (default: 'automatic') :param fpga_affinity: FPA Core affinity (default: 'automatic') :param fpga_reprogram: FPGA repogram command (default: '') :param profile_input: Input profile (default: '') :param profile_output: Output profile (default: '') :param scheduler_config: Scheduler configuration (default: '') :param external_adaptation: External adaptation [ True | False ] (default: False) :param propagate_virtual_environment: Propagate virtual environment [ True | False ] (default: False) :param mpi_worker: Use the MPI worker [ True | False ] (default: False) :param args: Positional arguments :param kwargs: Named arguments :return: Execution result """ # Let the Python binding know we are at master context.set_pycompss_context(context.MASTER) # Then we can import the appropriate start and stop functions from the API from pycompss.api.api import compss_start, compss_stop ############################################################## # INITIALIZATION ############################################################## # Initial dictionary with the user defined parameters all_vars = { 'log_level': log_level, 'debug': debug, 'o_c': o_c, 'graph': graph, 'trace': trace, 'monitor': monitor, 'project_xml': project_xml, 'resources_xml': resources_xml, 'summary': summary, 'task_execution': task_execution, 'storage_impl': storage_impl, 'storage_conf': storage_conf, 'streaming_backend': streaming_backend, 'streaming_master_name': streaming_master_name, 'streaming_master_port': streaming_master_port, 'task_count': task_count, 'app_name': app_name, 'uuid': uuid, 'base_log_dir': base_log_dir, 'specific_log_dir': specific_log_dir, 'extrae_cfg': extrae_cfg, 'comm': comm, 'conn': conn, 'master_name': master_name, 'master_port': master_port, 'scheduler': scheduler, 'jvm_workers': jvm_workers, 'cpu_affinity': cpu_affinity, 'gpu_affinity': gpu_affinity, 'fpga_affinity': fpga_affinity, 'fpga_reprogram': fpga_reprogram, 'profile_input': profile_input, 'profile_output': profile_output, 'scheduler_config': scheduler_config, 'external_adaptation': external_adaptation, 'propagate_virtual_environment': propagate_virtual_environment, 'mpi_worker': mpi_worker } # Check the provided flags flags, issues = check_flags(all_vars) if not flags: print_flag_issues(issues) return None # Prepare the environment env_vars = prepare_environment(False, o_c, storage_impl, app, debug, trace, mpi_worker) all_vars.update(env_vars) monitoring_vars = prepare_loglevel_graph_for_monitoring( monitor, graph, debug, log_level) all_vars.update(monitoring_vars) if RUNNING_IN_SUPERCOMPUTER: updated_vars = updated_variables_in_sc() all_vars.update(updated_vars) to_update = prepare_tracing_environment(all_vars['trace'], all_vars['extrae_lib'], all_vars['ld_library_path']) all_vars['trace'], all_vars['ld_library_path'] = to_update inf_vars = check_infrastructure_variables(all_vars['project_xml'], all_vars['resources_xml'], all_vars['compss_home'], all_vars['app_name'], all_vars['file_name'], all_vars['external_adaptation']) all_vars.update(inf_vars) create_init_config_file(**all_vars) ############################################################## # RUNTIME START ############################################################## # Runtime start compss_start(log_level, all_vars['trace'], True) # Setup logging binding_log_path = get_log_path() log_path = os.path.join(all_vars['compss_home'], 'Bindings', 'python', str(all_vars['major_version']), 'log') set_temporary_directory(binding_log_path) logging_cfg_file = get_logging_cfg_file(log_level) init_logging(os.path.join(log_path, logging_cfg_file), binding_log_path) logger = logging.getLogger("pycompss.runtime.launch") logger.debug('--- START ---') logger.debug('PyCOMPSs Log path: %s' % log_path) logger.debug("Starting storage") persistent_storage = master_init_storage(all_vars['storage_conf'], logger) logger.debug("Starting streaming") streaming = init_streaming(all_vars['streaming_backend'], all_vars['streaming_master_name'], all_vars['streaming_master_port']) saved_argv = sys.argv sys.argv = args # Execution: with event(APPLICATION_RUNNING_EVENT, master=True): if func is None or func == '__main__': if IS_PYTHON3: exec(open(app).read()) else: execfile(app) # noqa result = None else: if IS_PYTHON3: import importlib.util spec = importlib.util.spec_from_file_location( all_vars['file_name'], app) # noqa: E501 imported_module = importlib.util.module_from_spec(spec) spec.loader.exec_module(imported_module) # noqa else: import imp # noqa imported_module = imp.load_source(all_vars['file_name'], app) # noqa method_to_call = getattr(imported_module, func) result = method_to_call(*args, **kwargs) # Recover the system arguments sys.argv = saved_argv # Stop streaming if streaming: stop_streaming() # Stop persistent storage if persistent_storage: master_stop_storage(logger) logger.debug('--- END ---') ############################################################## # RUNTIME STOP ############################################################## # Stop runtime compss_stop() return result
def start( log_level='off', # type: str debug=False, # type: bool o_c=False, # type: bool graph=False, # type: bool trace=False, # type: bool monitor=None, # type: int project_xml=None, # type: str resources_xml=None, # type: str summary=False, # type: bool task_execution='compss', # type: str storage_impl=None, # type: str storage_conf=None, # type: str streaming_backend=None, # type: str streaming_master_name=None, # type: str streaming_master_port=None, # type: str task_count=50, # type: int app_name=INTERACTIVE_FILE_NAME, # type: str uuid=None, # type: str base_log_dir=None, # type: str specific_log_dir=None, # type: str extrae_cfg=None, # type: str comm='NIO', # type: str conn='es.bsc.compss.connectors.DefaultSSHConnector', # type: str master_name='', # type: str master_port='', # type: str scheduler='es.bsc.compss.scheduler.loadbalancing.LoadBalancingScheduler', # type: str # noqa: E501 jvm_workers='-Xms1024m,-Xmx1024m,-Xmn400m', # type: str cpu_affinity='automatic', # type: str gpu_affinity='automatic', # type: str fpga_affinity='automatic', # type: str fpga_reprogram='', # type: str profile_input='', # type: str profile_output='', # type: str scheduler_config='', # type: str external_adaptation=False, # type: bool propagate_virtual_environment=True, # type: bool mpi_worker=False, # type: bool verbose=False # type: bool ): # type: (...) -> None """ Start the runtime in interactive mode. :param log_level: Logging level [ 'trace'|'debug'|'info'|'api'|'off' ] (default: 'off') :param debug: Debug mode [ True | False ] (default: False) (overrides log-level) :param o_c: Objects to string conversion [ True|False ] (default: False) :param graph: Generate graph [ True|False ] (default: False) :param trace: Generate trace [ True|False|'scorep'|'arm-map'|'arm-ddt' ] (default: False) :param monitor: Monitor refresh rate (default: None) :param project_xml: Project xml file path (default: None) :param resources_xml: Resources xml file path (default: None) :param summary: Execution summary [ True | False ] (default: False) :param task_execution: Task execution (default: 'compss') :param storage_impl: Storage implementation path (default: None) :param storage_conf: Storage configuration file path (default: None) :param streaming_backend: Streaming backend (default: None) :param streaming_master_name: Streaming master name (default: None) :param streaming_master_port: Streaming master port (default: None) :param task_count: Task count (default: 50) :param app_name: Application name default: INTERACTIVE_FILE_NAME) :param uuid: UUId (default: None) :param base_log_dir: Base logging directory (default: None) :param specific_log_dir: Specific logging directory (default: None) :param extrae_cfg: Extrae configuration file path (default: None) :param comm: Communication library (default: NIO) :param conn: Connector (default: DefaultSSHConnector) :param master_name: Master Name (default: '') :param master_port: Master port (default: '') :param scheduler: Scheduler (see runcompss) (default: es.bsc.compss.scheduler.loadbalancing.LoadBalancingScheduler) # noqa :param jvm_workers: Java VM parameters (default: '-Xms1024m,-Xmx1024m,-Xmn400m') :param cpu_affinity: CPU Core affinity (default: 'automatic') :param gpu_affinity: GPU affinity (default: 'automatic') :param fpga_affinity: FPGA affinity (default: 'automatic') :param fpga_reprogram: FPGA repogram command (default: '') :param profile_input: Input profile (default: '') :param profile_output: Output profile (default: '') :param scheduler_config: Scheduler configuration (default: '') :param external_adaptation: External adaptation [ True|False ] (default: False) :param propagate_virtual_environment: Propagate virtual environment [ True|False ] # noqa (default: False) :param mpi_worker: Use the MPI worker [ True|False ] (default: False) :param verbose: Verbose mode [ True|False ] (default: False) :return: None """ # Export global variables global GRAPHING GRAPHING = graph __export_globals__() interactive_helpers.DEBUG = debug __show_flower__() # Let the Python binding know we are at master context.set_pycompss_context(context.MASTER) # Then we can import the appropriate start and stop functions from the API from pycompss.api.api import compss_start ############################################################## # INITIALIZATION ############################################################## # Initial dictionary with the user defined parameters all_vars = { 'log_level': log_level, 'debug': debug, 'o_c': o_c, 'graph': graph, 'trace': trace, 'monitor': monitor, 'project_xml': project_xml, 'resources_xml': resources_xml, 'summary': summary, 'task_execution': task_execution, 'storage_impl': storage_impl, 'storage_conf': storage_conf, 'streaming_backend': streaming_backend, 'streaming_master_name': streaming_master_name, 'streaming_master_port': streaming_master_port, 'task_count': task_count, 'app_name': app_name, 'uuid': uuid, 'base_log_dir': base_log_dir, 'specific_log_dir': specific_log_dir, 'extrae_cfg': extrae_cfg, 'comm': comm, 'conn': conn, 'master_name': master_name, 'master_port': master_port, 'scheduler': scheduler, 'jvm_workers': jvm_workers, 'cpu_affinity': cpu_affinity, 'gpu_affinity': gpu_affinity, 'fpga_affinity': fpga_affinity, 'fpga_reprogram': fpga_reprogram, 'profile_input': profile_input, 'profile_output': profile_output, 'scheduler_config': scheduler_config, 'external_adaptation': external_adaptation, 'propagate_virtual_environment': propagate_virtual_environment, 'mpi_worker': mpi_worker } # Check the provided flags flags, issues = check_flags(all_vars) if not flags: print_flag_issues(issues) return None # Prepare the environment env_vars = prepare_environment(True, o_c, storage_impl, 'undefined', debug, trace, mpi_worker) all_vars.update(env_vars) # Update the log level and graph values if monitoring is enabled monitoring_vars = prepare_loglevel_graph_for_monitoring( monitor, graph, debug, log_level) all_vars.update(monitoring_vars) # Check if running in supercomputer and update the variables accordingly # with the defined in the launcher and exported in environment variables. if RUNNING_IN_SUPERCOMPUTER: updated_vars = updated_variables_in_sc() if verbose: print("- Overridden project xml with: %s" % updated_vars['project_xml']) print("- Overridden resources xml with: %s" % updated_vars['resources_xml']) print("- Overridden master name with: %s" % updated_vars['master_name']) print("- Overridden master port with: %s" % updated_vars['master_port']) print("- Overridden uuid with: %s" % updated_vars['uuid']) print("- Overridden base log dir with: %s" % updated_vars['base_log_dir']) print("- Overridden specific log dir with: %s" % updated_vars['specific_log_dir']) print("- Overridden storage conf with: %s" % updated_vars['storage_conf']) print("- Overridden log level with: %s" % str(updated_vars['log_level'])) print("- Overridden debug with: %s" % str(updated_vars['debug'])) print("- Overridden trace with: %s" % str(updated_vars['trace'])) all_vars.update(updated_vars) # Update the tracing environment if set and set the appropriate trace # integer value tracing_vars = prepare_tracing_environment(all_vars['trace'], all_vars['extrae_lib'], all_vars['ld_library_path']) all_vars['trace'], all_vars['ld_library_path'] = tracing_vars # Update the infrastructure variables if necessary inf_vars = check_infrastructure_variables(all_vars['project_xml'], all_vars['resources_xml'], all_vars['compss_home'], all_vars['app_name'], all_vars['file_name'], all_vars['external_adaptation']) all_vars.update(inf_vars) # With all this information, create the configuration file for the # runtime start create_init_config_file(**all_vars) ############################################################## # RUNTIME START ############################################################## print("* - Starting COMPSs runtime... *") sys.stdout.flush() # Force flush compss_start(log_level, all_vars['trace'], True) global LOG_PATH LOG_PATH = get_log_path() set_temporary_directory(LOG_PATH) print("* - Log path : " + LOG_PATH) # Setup logging binding_log_path = get_log_path() log_path = os.path.join(all_vars['compss_home'], 'Bindings', 'python', str(all_vars['major_version']), 'log') set_temporary_directory(binding_log_path) logging_cfg_file = get_logging_cfg_file(log_level) init_logging(os.path.join(log_path, logging_cfg_file), binding_log_path) logger = logging.getLogger("pycompss.runtime.launch") __print_setup__(verbose, all_vars) logger.debug("--- START ---") logger.debug("PyCOMPSs Log path: %s" % LOG_PATH) logger.debug("Starting storage") global PERSISTENT_STORAGE PERSISTENT_STORAGE = master_init_storage(all_vars['storage_conf'], logger) logger.debug("Starting streaming") global STREAMING STREAMING = init_streaming(all_vars['streaming_backend'], all_vars['streaming_master_name'], all_vars['streaming_master_port']) # MAIN EXECUTION # let the user write an interactive application print("* - PyCOMPSs Runtime started... Have fun! *") print("******************************************************") # Emit the application start event (the 0 is in the stop function) emit_manual_event(APPLICATION_RUNNING_EVENT)
def compss_main(): # type: () -> None """ PyCOMPSs main function. General call: python $PYCOMPSS_HOME/pycompss/runtime/launch.py $log_level $PyObject_serialize $storage_conf $streaming_backend $streaming_master_name $streaming_master_port $fullAppPath $application_args :return: None """ global APP_PATH # Let the Python binding know we are at master context.set_pycompss_context(context.MASTER) # Then we can import the appropriate start and stop functions from the API from pycompss.api.api import compss_start, compss_stop # See parse_arguments, defined above # In order to avoid parsing user arguments, we are going to remove user # args from sys.argv user_sys_argv = sys.argv[9:] sys.argv = sys.argv[:9] args = parse_arguments() # We are done, now sys.argv must contain user args only sys.argv = [args.app_path] + user_sys_argv # Get log_level log_level = args.log_level # Setup tracing tracing = int(args.tracing) # Start the runtime compss_start(log_level, tracing, False) # Get object_conversion boolean set_object_conversion(args.object_conversion == 'true') # Get storage configuration at master storage_conf = args.storage_configuration # Get application execution path APP_PATH = args.app_path # Setup logging binding_log_path = get_log_path() log_path = os.path.join(os.getenv('COMPSS_HOME'), 'Bindings', 'python', str(_PYTHON_VERSION), 'log') set_temporary_directory(binding_log_path) logging_cfg_file = get_logging_cfg_file(log_level) init_logging(os.path.join(log_path, logging_cfg_file), binding_log_path) logger = logging.getLogger("pycompss.runtime.launch") # Get JVM options # jvm_opts = os.environ['JVM_OPTIONS_FILE'] # from pycompss.util.jvm.parser import convert_to_dict # opts = convert_to_dict(jvm_opts) # storage_conf = opts.get('-Dcompss.storage.conf') exit_code = 0 try: if __debug__: logger.debug('--- START ---') logger.debug('PyCOMPSs Log path: %s' % binding_log_path) # Start persistent storage persistent_storage = master_init_storage(storage_conf, logger) # Start streaming streaming = init_streaming(args.streaming_backend, args.streaming_master_name, args.streaming_master_port) # Show module warnings if __debug__: show_optional_module_warnings() # MAIN EXECUTION with event(APPLICATION_RUNNING_EVENT, master=True): if IS_PYTHON3: with open(APP_PATH) as f: exec(compile(f.read(), APP_PATH, 'exec'), globals()) else: execfile(APP_PATH, globals()) # MAIN EXECUTION # Stop streaming if streaming: stop_streaming() # Stop persistent storage if persistent_storage: master_stop_storage(logger) # End if __debug__: logger.debug('--- END ---') except SystemExit as e: if e.code != 0: print('[ ERROR ]: User program ended with exitcode %s.' % e.code) print('\t\tShutting down runtime...') exit_code = e.code except SerializerException: exit_code = 1 # If an object that can not be serialized has been used as a parameter. print("[ ERROR ]: Serialization exception") exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception(exc_type, exc_value, exc_traceback) for line in lines: if APP_PATH in line: print('[ ERROR ]: In: %s', line) exit_code = 1 except COMPSsException as e: # Any other exception occurred print("[ ERROR ]: A COMPSs exception occurred: " + str(e)) traceback.print_exc() exit_code = 0 # COMPSs exception is not considered an error except Exception as e: # Any other exception occurred print("[ ERROR ]: An exception occurred: " + str(e)) traceback.print_exc() exit_code = 1 finally: # Stop runtime compss_stop(exit_code) sys.stdout.flush() sys.stderr.flush() sys.exit(exit_code)
def compss_main(): ''' General call: python $PYCOMPSS_HOME/pycompss/runtime/launch.py $log_level $PyObject_serialize $storageConf $fullAppPath $application_args ''' global app_path # Start the runtime, see bindings commons compss_start() # See parse_arguments, defined above # In order to avoid parsing user arguments, we are going to remove user # args from sys.argv user_sys_argv = sys.argv[5:] sys.argv = sys.argv[:5] args = parse_arguments() # We are done, now sys.argv must contain user args only sys.argv = [args.app_path] + user_sys_argv # Get log_level log_level = args.log_level # Get object_conversion boolean binding.object_conversion = args.object_conversion == 'true' # Get storage configuration at master storage_conf = args.storage_configuration persistent_storage = False if storage_conf != 'null': persistent_storage = True from storage.api import init as init_storage from storage.api import finish as finish_storage # Get application execution path app_path = args.app_path binding_log_path = get_log_path() log_path = os.path.join(os.getenv('COMPSS_HOME'), 'Bindings', 'python', str(py_version), 'log') binding.temp_dir = mkdtemp(prefix='pycompss', dir=os.path.join(binding_log_path, 'tmpFiles/')) logging_cfg_file = get_logging_cfg_file(log_level) init_logging(os.path.join(log_path, logging_cfg_file), binding_log_path) if __debug__: logger = logging.getLogger('pycompss.runtime.launch') # Get JVM options jvm_opts = os.environ['JVM_OPTIONS_FILE'] opts = convert_to_dict(jvm_opts) # storage_conf = opts.get('-Dcompss.storage.conf') try: if __debug__: logger.debug('--- START ---') logger.debug('PyCOMPSs Log path: %s' % binding_log_path) if persistent_storage: if __debug__: logger.debug('Storage configuration file: %s' % storage_conf) init_storage(config_file_path=storage_conf) show_optional_module_warnings() # MAIN EXECUTION if sys.version_info >= (3, 0): exec(compile(open(app_path).read(), app_path, 'exec'), globals()) else: execfile(app_path, globals()) # MAIN EXECUTION if persistent_storage: finish_storage() if __debug__: logger.debug('--- END ---') except SystemExit as e: if e.code != 0: # Seems this is not happening print('[ ERROR ]: User program ended with exitcode %s.' % e.code) print('\t\tShutting down runtime...') except SerializerException: # If an object that can not be serialized has been used as a parameter. exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception(exc_type, exc_value, exc_traceback) for line in lines: if app_path in line: print('[ ERROR ]: In: %s', line) finally: compss_stop() sys.stdout.flush() sys.stderr.flush()
def start( log_level='off', o_c=False, debug=False, graph=False, trace=False, monitor=None, project_xml=None, resources_xml=None, summary=False, taskExecution='compss', storageConf=None, taskCount=50, appName='Interactive', uuid=None, baseLogDir=None, specificLogDir=None, extraeCfg=None, comm='NIO', conn='es.bsc.compss.connectors.DefaultSSHConnector', masterName='', masterPort='', scheduler='es.bsc.compss.scheduler.loadBalancingScheduler.LoadBalancingScheduler', jvmWorkers='-Xms1024m,-Xmx1024m,-Xmn400m', cpuAffinity='automatic', gpuAffinity='automatic', profileInput='', profileOutput='', scheduler_config='', external_adaptation=False, propagate_virtual_environment=False, verbose=False): launchPath = os.path.dirname(os.path.realpath(__file__)) # compss_home = launchPath without the last 4 folders: # Bindings/python/version/pycompss compss_home = os.path.sep.join(launchPath.split(os.path.sep)[:-4]) os.environ['COMPSS_HOME'] = compss_home # Get environment variables cp = os.getcwd() + '/' pythonPath = os.environ['PYTHONPATH'] classPath = os.environ['CLASSPATH'] ld_library_path = os.environ['LD_LIBRARY_PATH'] # Set extrae dependencies extrae_home = compss_home + '/Dependencies/extrae' extrae_lib = extrae_home + '/lib' os.environ['EXTRAE_HOME'] = extrae_home os.environ['LD_LIBRARY_PATH'] = extrae_lib + ':' + ld_library_path if trace is False: trace = 0 elif trace == 'basic' or trace is True: trace = 1 os.environ['LD_PRELOAD'] = extrae_lib + '/libpttrace.so' elif trace == 'advanced': trace = 2 os.environ['LD_PRELOAD'] = extrae_lib + '/libpttrace.so' else: print('ERROR: Wrong tracing parameter ( [ True | basic ] | \ advanced | False)') return -1 if monitor is not None: # Enable the graph if the monitoring is enabled graph = True global graphing graphing = graph global tracing tracing = trace global monitoring monitoring = monitor __export_globals__() print("********************************************************") print("*************** PyCOMPSs Interactive *******************") print("********************************************************") print("* .-~~-.--. ____ ____ *") print("* : ) |___ \ |___ \ *") print("* .~ ~ -.\ /.- ~~ . __) | __) | *") print("* > `. .' < / __/ _ / __/ *") print("* ( .- -. ) |_____| |_| |_____| *") print("* `- -.-~ `- -' ~-.- -' *") print("* ( : ) _ _ .-: *") print("* ~--. : .--~ .-~ .-~ } *") print("* ~-.-^-.-~ \_ .~ .-~ .~ *") print("* \ \ ' \ '_ _ -~ *") print("* \`.\`. // *") print("* . - ~ ~-.__\`.\`-.// *") print("* .-~ . - ~ }~ ~ ~-.~-. *") print("* .' .-~ .-~ :/~-.~-./: *") print("* /_~_ _ . - ~ ~-.~-._ *") print("* ~-.< *") print("********************************************************") # print("* .-~~-.--. ____ _____ *") # print("* : ) |___ \ |___ / *") # print("* .~ ~ -.\ /.- ~~ . __) | |_ \ *") # print("* > `. .' < / __/ _ __) | *") # print("* ( .- -. ) |_____| |_| |___/ *") ############################################################## # INITIALIZATION ############################################################## # Build a dictionary with all variables needed for initializing the runtime config = {} config['compss_home'] = compss_home config['debug'] = debug if project_xml is None: projPath = 'Runtime/configuration/xml/projects/default_project.xml' config['project_xml'] = compss_home + os.path.sep + projPath else: config['project_xml'] = project_xml if resources_xml is None: resPath = 'Runtime/configuration/xml/resources/default_resources.xml' config['resources_xml'] = compss_home + os.path.sep + resPath else: config['resources_xml'] = resources_xml config['summary'] = summary config['taskExecution'] = taskExecution config['storageConf'] = storageConf config['taskCount'] = taskCount if appName is None: config['appName'] = 'Interactive' else: config['appName'] = appName config['uuid'] = uuid config['baseLogDir'] = baseLogDir config['specificLogDir'] = specificLogDir config['graph'] = graph config['monitor'] = monitor config['trace'] = trace config['extraeCfg'] = extraeCfg config['comm'] = comm config['conn'] = conn config['masterName'] = masterName config['masterPort'] = masterPort config['scheduler'] = scheduler config['cp'] = cp config['classpath'] = classPath config['jvmWorkers'] = jvmWorkers config['pythonPath'] = pythonPath config['cpuAffinity'] = cpuAffinity config['gpuAffinity'] = gpuAffinity config['profileInput'] = profileInput config['profileOutput'] = profileOutput config['scheduler_config'] = scheduler_config if external_adaptation: config['external_adaptation'] = 'true' else: config['external_adaptation'] = 'false' major_version = sys.version_info[0] python_interpreter = 'python' + str(major_version) config['python_interpreter'] = python_interpreter config['python_version'] = str(major_version) if 'VIRTUAL_ENV' in os.environ: # Running within a virtual environment python_virtual_environment = os.environ['VIRTUAL_ENV'] else: python_virtual_environment = 'null' config['python_virtual_environment'] = python_virtual_environment config[ 'python_propagate_virtual_environment'] = propagate_virtual_environment initialize_compss(config) ############################################################## # RUNTIME START ############################################################## print("* - Starting COMPSs runtime... *") compss_start() if o_c is True: # set cross-module variable binding.object_conversion = True else: # set cross-module variable binding.object_conversion = False # Remove launch.py, log_level and object_conversion from sys.argv, # It will be inherited by the app through execfile # sys.argv = sys.argv[3:] # Get application execution path # app_path = sys.argv[0] # not needed in interactive mode global log_path log_path = get_log_path() binding.temp_dir = mkdtemp(prefix='pycompss', dir=log_path + '/tmpFiles/') print("* - Log path : " + log_path) # Logging setup if log_level == "debug": jsonPath = '/Bindings/python/' + str( major_version) + '/log/logging.json.debug' init_logging(os.getenv('COMPSS_HOME') + jsonPath, log_path) elif log_level == "info": jsonPath = '/Bindings/python/' + str( major_version) + '/log/logging.json.off' init_logging(os.getenv('COMPSS_HOME') + jsonPath, log_path) elif log_level == "off": jsonPath = '/Bindings/python/' + str( major_version) + '/log/logging.json.off' init_logging(os.getenv('COMPSS_HOME') + jsonPath, log_path) else: # Default jsonPath = '/Bindings/python/' + str( major_version) + '/log/logging.json' init_logging(os.getenv('COMPSS_HOME') + jsonPath, log_path) logger = logging.getLogger("pycompss.runtime.launch") printSetup(verbose, log_level, o_c, debug, graph, trace, monitor, project_xml, resources_xml, summary, taskExecution, storageConf, taskCount, appName, uuid, baseLogDir, specificLogDir, extraeCfg, comm, conn, masterName, masterPort, scheduler, jvmWorkers, cpuAffinity, gpuAffinity, profileInput, profileOutput, scheduler_config, external_adaptation, python_interpreter, major_version, python_virtual_environment, propagate_virtual_environment) logger.debug("--- START ---") logger.debug("PyCOMPSs Log path: %s" % log_path) if storageConf is not None: logger.debug("Storage configuration file: %s" % storageConf) from storage.api import init as initStorage initStorage(config_file_path=storageConf) global persistent_storage persistent_storage = True # MAIN EXECUTION # let the user write an interactive application print("* - PyCOMPSs Runtime started... Have fun! *") print("********************************************************")
def launch_pycompss_application( app, func, log_level='off', o_c=False, debug=False, graph=False, trace=False, monitor=None, project_xml=None, resources_xml=None, summary=False, task_execution='compss', storage_impl=None, storage_conf=None, task_count=50, app_name=None, uuid=None, base_log_dir=None, specific_log_dir=None, extrae_cfg=None, comm='NIO', conn='es.bsc.compss.connectors.DefaultSSHConnector', master_name='', master_port='', scheduler='es.bsc.compss.scheduler.loadBalancingScheduler.LoadBalancingScheduler', jvm_workers='-Xms1024m,-Xmx1024m,-Xmn400m', obj_conv=False, cpu_affinity='automatic', gpu_affinity='automatic', fpga_affinity='automatic', fpga_reprogram='', profile_input='', profile_output='', scheduler_config='', external_adaptation=False, python_propagate_virtual_environment=True, *args, **kwargs): """ Launch pycompss application. :param app: Application path :param func: Function :param args: Arguments :param kwargs: Keyword arguments :param log_level: Logging level [ 'on' | 'off'] (default: 'off') :param o_c: Objects to string conversion [ True | False ] (default: False) :param debug: Debug mode [ True | False ] (default: False) :param graph: Generate graph [ True | False ] (default: False) :param trace: Generate trace [ True | False ] (default: False) :param monitor: Monitor refresh rate (default: None) :param project_xml: Project xml file path :param resources_xml: Resources xml file path :param summary: Execution summary [ True | False ] (default: False) :param task_execution: Task execution (default: 'compss') :param storage_impl: Storage implementation path :param storage_conf: Storage configuration file path :param task_count: Task count (default: 50) :param app_name: Application name (default: Interactive_date) :param uuid: UUId :param base_log_dir: Base logging directory :param specific_log_dir: Specific logging directory :param extrae_cfg: Extrae configuration file path :param comm: Communication library (default: NIO) :param conn: Connector (default: DefaultSSHConnector) :param master_name: Master Name (default: '') :param master_port: Master port (default: '') :param scheduler: Scheduler (default: LoadBalancingScheduler) :param jvm_workers: Java VM parameters (default: '-Xms1024m,-Xmx1024m,-Xmn400m') :param cpu_affinity: CPU Core affinity (default: 'automatic') :param gpu_affinity: GPU Core affinity (default: 'automatic') :param fpga_affinity: FPA Core affinity (default: 'automatic') :param fpga_reprogram: FPGA repogram command (default: '') :param profile_input: Input profile (default: '') :param profile_output: Output profile (default: '') :param scheduler_config: Scheduler configuration (default: '') :param external_adaptation: External adaptation [ True | False ] (default: False) :param python_propagate_virtual_environment: Propagate virtual environment [ True | False ] (default: False) :return: Execution result """ global app_path # Let the Python binding know we are at master set_pycompss_context('MASTER') launch_path = os.path.dirname(os.path.abspath(__file__)) # compss_home = launch_path without the last 4 folders: # (Bindings/python/pycompss/runtime) compss_home = os.path.sep.join(launch_path.split(os.path.sep)[:-4]) # Grab the existing PYTHONPATH and CLASSPATH values pythonpath = os.environ['PYTHONPATH'] classpath = os.environ['CLASSPATH'] ld_library_path = os.environ['LD_LIBRARY_PATH'] # Enable/Disable object to string conversion binding.object_conversion = obj_conv # Get the filename and its path. file_name = os.path.splitext(os.path.basename(app))[0] cp = os.path.dirname(app) # Set storage classpath if storage_impl: if storage_impl == 'redis': cp = cp + ':' + compss_home + '/Tools/storage/redis/compss-redisPSCO.jar' else: cp = cp + ':' + storage_impl if RUNNING_IN_SUPERCOMPUTER: # Since the deployment in supercomputers is done through the use of enqueue_compss # and consequently launch_compss - the project and resources xmls are already created project_xml, resources_xml = get_xmls() # It also exported some environment variables that we need here master_name = get_master_node() master_port = get_master_port() uuid = get_uuid() base_log_dir = get_base_log_dir() specific_log_dir = get_specific_log_dir() storage_conf = get_storage_conf() # Override debug considering the parameter defined in pycompss_interactive_sc script # and exported by launch_compss log_level = get_log_level() if log_level == 'off': debug = False else: debug = True # Override tracing considering the parameter defined in pycompss_interactive_sc script # and exported by launch_compss if get_tracing(): trace = 1 else: trace = 0 # Build a dictionary with all variables needed for initializing the runtime. config = dict() config['compss_home'] = compss_home config['debug'] = debug if project_xml is None: project_xml = 'Runtime/configuration/xml/projects/default_project.xml' config['project_xml'] = compss_home + os.path.sep + project_xml else: config['project_xml'] = project_xml if resources_xml is None: resources_xml = 'Runtime/configuration/xml/resources/default_resources.xml' config['resources_xml'] = compss_home + os.path.sep + resources_xml else: config['resources_xml'] = resources_xml config['summary'] = summary config['task_execution'] = task_execution config['storage_conf'] = storage_conf config['task_count'] = task_count if app_name is None: config['app_name'] = file_name else: config['app_name'] = app_name config['uuid'] = uuid config['base_log_dir'] = base_log_dir config['specific_log_dir'] = specific_log_dir config['graph'] = graph config['monitor'] = monitor config['trace'] = trace config['extrae_cfg'] = extrae_cfg config['comm'] = comm config['conn'] = conn config['master_name'] = master_name config['master_port'] = master_port config['scheduler'] = scheduler config['cp'] = cp config['classpath'] = classpath config['ld_library_path'] = ld_library_path config['jvm_workers'] = jvm_workers config['pythonpath'] = pythonpath config['cpu_affinity'] = cpu_affinity config['gpu_affinity'] = gpu_affinity config['fpga_affinity'] = fpga_affinity config['fpga_reprogram'] = fpga_reprogram config['profile_input'] = profile_input config['profile_output'] = profile_output config['scheduler_config'] = scheduler_config if external_adaptation: config['external_adaptation'] = 'true' else: config['external_adaptation'] = 'false' major_version = str(sys.version_info[0]) config['python_interpreter'] = 'python' + major_version config['python_version'] = major_version if 'VIRTUAL_ENV' in os.environ: # Running within a virtual environment python_virtual_environment = os.environ['VIRTUAL_ENV'] else: python_virtual_environment = 'null' config['python_virtual_environment'] = python_virtual_environment config[ 'python_propagate_virtual_environment'] = python_propagate_virtual_environment initialize_compss(config) # Runtime start compss_start() # Configure logging app_path = app log_path = get_log_path() # Logging setup if debug or log_level == "debug": json_path = '/Bindings/python/' + major_version + '/log/logging.json.debug' init_logging(compss_home + json_path, log_path) elif log_level == "info": json_path = '/Bindings/python/' + major_version + '/log/logging.json.off' init_logging(compss_home + json_path, log_path) elif log_level == "off": json_path = '/Bindings/python/' + major_version + '/log/logging.json.off' init_logging(compss_home + json_path, log_path) else: # Default json_path = '/Bindings/python/' + str( major_version) + '/log/logging.json' init_logging(compss_home + json_path, log_path) logger = logging.getLogger("pycompss.runtime.launch") logger.debug('--- START ---') logger.debug('PyCOMPSs Log path: %s' % log_path) saved_argv = sys.argv sys.argv = args # Execution: if func is None or func == '__main__': execfile(app) result = None else: import imp imported_module = imp.load_source(file_name, app) method_to_call = getattr(imported_module, func) result = method_to_call(*args, **kwargs) # Recover the system arguments sys.argv = saved_argv logger.debug('--- END ---') compss_stop() return result
def launch_pycompss_application( app, func, args=[], kwargs={}, log_level='off', o_c=False, debug=False, graph=False, trace=False, monitor=None, project_xml=None, resources_xml=None, summary=False, taskExecution='compss', storageConf=None, taskCount=50, appName=None, uuid=None, baseLogDir=None, specificLogDir=None, extraeCfg=None, comm='NIO', conn='es.bsc.compss.connectors.DefaultSSHConnector', masterName='', masterPort='', scheduler='es.bsc.compss.scheduler.loadBalancingScheduler.LoadBalancingScheduler', jvmWorkers='-Xms1024m,-Xmx1024m,-Xmn400m', obj_conv=False, cpuAffinity='automatic', gpuAffinity='automatic', profileInput='', profileOutput='', scheduler_config='', external_adaptation=False, python_propagate_virtual_environment=True): global app_path launchPath = os.path.dirname(os.path.abspath(__file__)) # compss_home = launchPath without the last 4 folders: # (Bindings/python/pycompss/runtime) compss_home = os.path.sep.join(launchPath.split(os.path.sep)[:-4]) # Grab the existing PYTHONPATH and CLASSPATH values pythonpath = os.environ['PYTHONPATH'] classpath = os.environ['CLASSPATH'] # Enable/Disable object to string conversion binding.object_conversion = obj_conv # Get the filename and its path. file_name = os.path.splitext(os.path.basename(app))[0] cp = os.path.dirname(app) # Build a dictionary with all variables needed for initializing the runtime. config = {} config['compss_home'] = compss_home config['debug'] = debug if project_xml is None: projXml = 'Runtime/configuration/xml/projects/default_project.xml' config['project_xml'] = compss_home + os.path.sep + projXml else: config['project_xml'] = project_xml if resources_xml is None: resXml = 'Runtime/configuration/xml/resources/default_resources.xml' config['resources_xml'] = compss_home + os.path.sep + resXml else: config['resources_xml'] = resources_xml config['summary'] = summary config['taskExecution'] = taskExecution config['storageConf'] = storageConf config['taskCount'] = taskCount if appName is None: config['appName'] = file_name else: config['appName'] = appName config['uuid'] = uuid config['baseLogDir'] = baseLogDir config['specificLogDir'] = specificLogDir config['graph'] = graph config['monitor'] = monitor config['trace'] = trace config['extraeCfg'] = extraeCfg config['comm'] = comm config['conn'] = conn config['masterName'] = masterName config['masterPort'] = masterPort config['scheduler'] = scheduler config['cp'] = cp config['classpath'] = classpath config['jvmWorkers'] = jvmWorkers config['pythonPath'] = pythonpath config['cpuAffinity'] = cpuAffinity config['gpuAffinity'] = gpuAffinity config['profileInput'] = profileInput config['profileOutput'] = profileOutput config['scheduler_config'] = scheduler_config if external_adaptation: config['external_adaptation'] = 'true' else: config['external_adaptation'] = 'false' config['python_interpreter'] = 'python' + str(sys.version_info[0]) config['python_version'] = str(sys.version_info[0]) if 'VIRTUAL_ENV' in os.environ: # Running within a virtual environment python_virtual_environment = os.environ['VIRTUAL_ENV'] else: python_virtual_environment = 'null' config['python_virtual_environment'] = python_virtual_environment config[ 'python_propagate_virtual_environment'] = python_propagate_virtual_environment initialize_compss(config) # Runtime start compss_start() # Configure logging app_path = app log_path = get_log_path() if debug: # DEBUG init_logging(compss_home + '/Bindings/python/log/logging.json.debug', log_path) else: # NO DEBUG init_logging(compss_home + '/Bindings/python/log/logging.json', log_path) logger = logging.getLogger("pycompss.runtime.launch") logger.debug('--- START ---') logger.debug('PyCOMPSs Log path: %s' % log_path) saved_argv = sys.argv sys.argv = args # Execution: if func is None or func == '__main__': result = execfile(app) else: import imp imported_module = imp.load_source(file_name, app) methodToCall = getattr(imported_module, func) result = methodToCall(*args, **kwargs) # Recover the system arguments sys.argv = saved_argv logger.debug('--- END ---') compss_stop() return result
def start( log_level="off", # type: str debug=False, # type: bool o_c=False, # type: bool graph=False, # type: bool trace=False, # type: bool monitor=-1, # type: int project_xml="", # type: str resources_xml="", # type: str summary=False, # type: bool task_execution="compss", # type: str storage_impl="", # type: str storage_conf="", # type: str streaming_backend="", # type: str streaming_master_name="", # type: str streaming_master_port="", # type: str task_count=50, # type: int app_name=INTERACTIVE_FILE_NAME, # type: str uuid="", # type: str base_log_dir="", # type: str specific_log_dir="", # type: str extrae_cfg="", # type: str comm="NIO", # type: str conn=DEFAULT_CONN, # type: str master_name="", # type: str master_port="", # type: str scheduler=DEFAULT_SCHED, # type: str jvm_workers=DEFAULT_JVM_WORKERS, # type: str cpu_affinity="automatic", # type: str gpu_affinity="automatic", # type: str fpga_affinity="automatic", # type: str fpga_reprogram="", # type: str profile_input="", # type: str profile_output="", # type: str scheduler_config="", # type: str external_adaptation=False, # type: bool propagate_virtual_environment=True, # type: bool mpi_worker=False, # type: bool worker_cache=False, # type: typing.Union[bool, str] shutdown_in_node_failure=False, # type: bool io_executors=0, # type: int env_script="", # type: str reuse_on_block=True, # type: bool nested_enabled=False, # type: bool tracing_task_dependencies=False, # type: bool trace_label="", # type: str extrae_cfg_python="", # type: str wcl=0, # type: int cache_profiler=False, # type: bool verbose=False # type: bool ): # type: (...) -> None """ Start the runtime in interactive mode. :param log_level: Logging level [ "trace"|"debug"|"info"|"api"|"off" ] (default: "off") :param debug: Debug mode [ True | False ] (default: False) (overrides log-level) :param o_c: Objects to string conversion [ True|False ] (default: False) :param graph: Generate graph [ True|False ] (default: False) :param trace: Generate trace [ True|False|"scorep"|"arm-map"|"arm-ddt" ] (default: False) :param monitor: Monitor refresh rate (default: None) :param project_xml: Project xml file path (default: None) :param resources_xml: Resources xml file path (default: None) :param summary: Execution summary [ True | False ] (default: False) :param task_execution: Task execution (default: "compss") :param storage_impl: Storage implementation path (default: None) :param storage_conf: Storage configuration file path (default: None) :param streaming_backend: Streaming backend (default: None) :param streaming_master_name: Streaming master name (default: None) :param streaming_master_port: Streaming master port (default: None) :param task_count: Task count (default: 50) :param app_name: Application name default: INTERACTIVE_FILE_NAME) :param uuid: UUId (default: None) :param base_log_dir: Base logging directory (default: None) :param specific_log_dir: Specific logging directory (default: None) :param extrae_cfg: Extrae configuration file path (default: None) :param comm: Communication library (default: NIO) :param conn: Connector (default: DefaultSSHConnector) :param master_name: Master Name (default: "") :param master_port: Master port (default: "") :param scheduler: Scheduler (see runcompss) (default: es.bsc.compss.scheduler.loadbalancing.LoadBalancingScheduler) # noqa: E501 :param jvm_workers: Java VM parameters (default: "-Xms1024m,-Xmx1024m,-Xmn400m") :param cpu_affinity: CPU Core affinity (default: "automatic") :param gpu_affinity: GPU affinity (default: "automatic") :param fpga_affinity: FPGA affinity (default: "automatic") :param fpga_reprogram: FPGA repogram command (default: "") :param profile_input: Input profile (default: "") :param profile_output: Output profile (default: "") :param scheduler_config: Scheduler configuration (default: "") :param external_adaptation: External adaptation [ True|False ] (default: False) :param propagate_virtual_environment: Propagate virtual environment [ True|False ] # noqa: E501 (default: False) :param mpi_worker: Use the MPI worker [ True|False ] (default: False) :param worker_cache: Use the worker cache [ True | int(size) | False] (default: False) :param shutdown_in_node_failure: Shutdown in node failure [ True | False] (default: False) :param io_executors: <Integer> Number of IO executors :param env_script: <String> Environment script to be sourced in workers :param reuse_on_block: Reuse on block [ True | False] (default: True) :param nested_enabled: Nested enabled [ True | False] (default: True) :param tracing_task_dependencies: Include task dependencies in trace [ True | False] (default: False) :param trace_label: <String> Add trace label :param extrae_cfg_python: <String> Extrae configuration file for the workers :param wcl: <Integer> Wall clock limit. Stops the runtime if reached. 0 means forever. :param cache_profiler: Use the cache profiler [ True | False] (default: False) :param verbose: Verbose mode [ True|False ] (default: False) :return: None """ # Export global variables global GRAPHING if context.in_pycompss(): print("The runtime is already running") return None GRAPHING = graph __export_globals__() interactive_helpers.DEBUG = debug if debug: log_level = "debug" __show_flower__() # Let the Python binding know we are at master context.set_pycompss_context(context.MASTER) # Then we can import the appropriate start and stop functions from the API from pycompss.api.api import compss_start ############################################################## # INITIALIZATION ############################################################## # Initial dictionary with the user defined parameters all_vars = parameters_to_dict( log_level, debug, o_c, graph, trace, monitor, project_xml, resources_xml, summary, task_execution, storage_impl, storage_conf, streaming_backend, streaming_master_name, streaming_master_port, task_count, app_name, uuid, base_log_dir, specific_log_dir, extrae_cfg, comm, conn, master_name, master_port, scheduler, jvm_workers, cpu_affinity, gpu_affinity, fpga_affinity, fpga_reprogram, profile_input, profile_output, scheduler_config, external_adaptation, propagate_virtual_environment, mpi_worker, worker_cache, shutdown_in_node_failure, io_executors, env_script, reuse_on_block, nested_enabled, tracing_task_dependencies, trace_label, extrae_cfg_python, wcl, cache_profiler) # Save all vars in global current flags so that events.py can restart # the notebook with the same flags export_current_flags(all_vars) # Check the provided flags flags, issues = check_flags(all_vars) if not flags: print_flag_issues(issues) return None # Prepare the environment env_vars = prepare_environment(True, o_c, storage_impl, "undefined", debug, trace, mpi_worker) all_vars.update(env_vars) # Update the log level and graph values if monitoring is enabled monitoring_vars = prepare_loglevel_graph_for_monitoring( monitor, graph, debug, log_level) all_vars.update(monitoring_vars) # Check if running in supercomputer and update the variables accordingly # with the defined in the launcher and exported in environment variables. if RUNNING_IN_SUPERCOMPUTER: updated_vars = updated_variables_in_sc() if verbose: print("- Overridden project xml with: %s" % updated_vars["project_xml"]) print("- Overridden resources xml with: %s" % updated_vars["resources_xml"]) print("- Overridden master name with: %s" % updated_vars["master_name"]) print("- Overridden master port with: %s" % updated_vars["master_port"]) print("- Overridden uuid with: %s" % updated_vars["uuid"]) print("- Overridden base log dir with: %s" % updated_vars["base_log_dir"]) print("- Overridden specific log dir with: %s" % updated_vars["specific_log_dir"]) print("- Overridden storage conf with: %s" % updated_vars["storage_conf"]) print("- Overridden log level with: %s" % str(updated_vars["log_level"])) print("- Overridden debug with: %s" % str(updated_vars["debug"])) print("- Overridden trace with: %s" % str(updated_vars["trace"])) all_vars.update(updated_vars) # Update the tracing environment if set and set the appropriate trace # integer value tracing_vars = prepare_tracing_environment(all_vars["trace"], all_vars["extrae_lib"], all_vars["ld_library_path"]) all_vars["trace"], all_vars["ld_library_path"] = tracing_vars # Update the infrastructure variables if necessary inf_vars = check_infrastructure_variables(all_vars["project_xml"], all_vars["resources_xml"], all_vars["compss_home"], all_vars["app_name"], all_vars["file_name"], all_vars["external_adaptation"]) all_vars.update(inf_vars) # With all this information, create the configuration file for the # runtime start create_init_config_file(**all_vars) # Start the event manager (ipython hooks) ipython = globals()["__builtins__"]["get_ipython"]() setup_event_manager(ipython) ############################################################## # RUNTIME START ############################################################## print("* - Starting COMPSs runtime... *") sys.stdout.flush() # Force flush compss_start(log_level, all_vars["trace"], True) global LOG_PATH LOG_PATH = get_log_path() set_temporary_directory(LOG_PATH) print("* - Log path : " + LOG_PATH) # Setup logging binding_log_path = get_log_path() log_path = os.path.join(all_vars["compss_home"], "Bindings", "python", str(all_vars["major_version"]), "log") set_temporary_directory(binding_log_path) logging_cfg_file = get_logging_cfg_file(log_level) init_logging(os.path.join(log_path, logging_cfg_file), binding_log_path) logger = logging.getLogger("pycompss.runtime.launch") __print_setup__(verbose, all_vars) logger.debug("--- START ---") logger.debug("PyCOMPSs Log path: %s" % LOG_PATH) logger.debug("Starting storage") global PERSISTENT_STORAGE PERSISTENT_STORAGE = master_init_storage(all_vars["storage_conf"], logger) logger.debug("Starting streaming") global STREAMING STREAMING = init_streaming(all_vars["streaming_backend"], all_vars["streaming_master_name"], all_vars["streaming_master_port"]) # Start monitoring the stdout and stderr STDW.start_watching() # MAIN EXECUTION # let the user write an interactive application print("* - PyCOMPSs Runtime started... Have fun! *") print(LINE_SEPARATOR) # Emit the application start event (the 0 is in the stop function) emit_manual_event(APPLICATION_RUNNING_EVENT)
def start( log_level='off', o_c=False, debug=False, graph=False, trace=False, monitor=None, project_xml=None, resources_xml=None, summary=False, task_execution='compss', storage_impl=None, storage_conf=None, task_count=50, app_name='Interactive', uuid=None, base_log_dir=None, specific_log_dir=None, extrae_cfg=None, comm='NIO', conn='es.bsc.compss.connectors.DefaultSSHConnector', master_name='', master_port='', scheduler='es.bsc.compss.scheduler.loadBalancingScheduler.LoadBalancingScheduler', jvm_workers='-Xms1024m,-Xmx1024m,-Xmn400m', cpu_affinity='automatic', gpu_affinity='automatic', fpga_affinity='automatic', fpga_reprogram='', profile_input='', profile_output='', scheduler_config='', external_adaptation=False, propagate_virtual_environment=False, verbose=False): """ Start the runtime in interactive mode. :param log_level: Logging level [ 'on' | 'off'] (default: 'off') :param o_c: Objects to string conversion [ True | False ] (default: False) :param debug: Debug mode [ True | False ] (default: False) :param graph: Generate graph [ True | False ] (default: False) :param trace: Generate trace [ True | False ] (default: False) :param monitor: Monitor refresh rate (default: None) :param project_xml: Project xml file path :param resources_xml: Resources xml file path :param summary: Execution summary [ True | False ] (default: False) :param task_execution: Task execution (default: 'compss') :param storage_impl: Storage implementation path :param storage_conf: Storage configuration file path :param task_count: Task count (default: 50) :param app_name: Application name (default: Interactive_date) :param uuid: UUId :param base_log_dir: Base logging directory :param specific_log_dir: Specific logging directory :param extrae_cfg: Extrae configuration file path :param comm: Communication library (default: NIO) :param conn: Connector (default: DefaultSSHConnector) :param master_name: Master Name (default: '') :param master_port: Master port (default: '') :param scheduler: Scheduler (default: LoadBalancingScheduler) :param jvm_workers: Java VM parameters (default: '-Xms1024m,-Xmx1024m,-Xmn400m') :param cpu_affinity: CPU Core affinity (default: 'automatic') :param gpu_affinity: GPU Core affinity (default: 'automatic') :param fpga_affinity: FPA Core affinity (default: 'automatic') :param fpga_reprogram: FPGA repogram command (default: '') :param profile_input: Input profile (default: '') :param profile_output: Output profile (default: '') :param scheduler_config: Scheduler configuration (default: '') :param external_adaptation: External adaptation [ True | False ] (default: False) :param propagate_virtual_environment: Propagate virtual environment [ True | False ] (default: False) :param verbose: Verbose mode [ True | False ] (default: False) :return: None """ # Let the Python binding know we are at master set_pycompss_context('MASTER') # Prepare the environment launch_path = os.path.dirname(os.path.realpath(__file__)) # compss_home = launch_path without the last 4 folders: # Bindings/python/version/pycompss compss_home = os.path.sep.join(launch_path.split(os.path.sep)[:-4]) os.environ['COMPSS_HOME'] = compss_home # Get environment variables cp = os.getcwd() + '/' pythonpath = os.environ['PYTHONPATH'] classpath = os.environ['CLASSPATH'] ld_library_path = os.environ['LD_LIBRARY_PATH'] # Set storage classpath if storage_impl: if storage_impl == 'redis': cp = cp + ':' + compss_home + '/Tools/storage/redis/compss-redisPSCO.jar' else: cp = cp + ':' + storage_impl # Set extrae dependencies if not "EXTRAE_HOME" in os.environ: # It can be defined by the user or by launch_compss when running in Supercomputer extrae_home = compss_home + '/Dependencies/extrae' os.environ['EXTRAE_HOME'] = extrae_home else: extrae_home = os.environ['EXTRAE_HOME'] extrae_lib = extrae_home + '/lib' # Include extrae into ld_library_path ld_library_path = extrae_lib + ':' + ld_library_path os.environ['LD_LIBRARY_PATH'] = ld_library_path if monitor is not None: # Enable the graph if the monitoring is enabled graph = True # Export global variables global graphing graphing = graph __export_globals__() print("******************************************************") print("*************** PyCOMPSs Interactive *****************") print("******************************************************") print("* .-~~-.--. ____ ____ *") print("* : ) |___ \ |___ \ *") print("* .~ ~ -.\ /.- ~~ . __) | __) | *") print("* > `. .' < / __/ _ ___) | *") print("* ( .- -. ) |_____| |_| |____/ *") print("* `- -.-~ `- -' ~-.- -' *") print("* ( : ) _ _ .-: *") print("* ~--. : .--~ .-~ .-~ } *") print("* ~-.-^-.-~ \_ .~ .-~ .~ *") print("* \ \ ' \ '_ _ -~ *") print("* \`.\`. // *") print("* . - ~ ~-.__\`.\`-.// *") print("* .-~ . - ~ }~ ~ ~-.~-. *") print("* .' .-~ .-~ :/~-.~-./: *") print("* /_~_ _ . - ~ ~-.~-._ *") print("* ~-.< *") print("******************************************************") ############################################################## # INITIALIZATION ############################################################## if RUNNING_IN_SUPERCOMPUTER: # Since the deployment in supercomputers is done through the use of enqueue_compss # and consequently launch_compss - the project and resources xmls are already created project_xml, resources_xml = get_xmls() # It also exported some environment variables that we need here master_name = get_master_node() master_port = get_master_port() uuid = get_uuid() base_log_dir = get_base_log_dir() specific_log_dir = get_specific_log_dir() storage_conf = get_storage_conf() # Override debug considering the parameter defined in pycompss_interactive_sc script # and exported by launch_compss log_level = get_log_level() if log_level == 'off': debug = False else: debug = True # Override tracing considering the parameter defined in pycompss_interactive_sc script # and exported by launch_compss trace = get_tracing() if verbose: print("- Overridden project xml with: " + project_xml) print("- Overridden resources xml with: " + resources_xml) print("- Overridden master name with: " + master_name) print("- Overridden master port with: " + master_port) print("- Overridden uuid with: " + uuid) print("- Overridden base log dir with: " + base_log_dir) print("- Overridden specific log dir with: " + specific_log_dir) print("- Overridden storage conf with: " + storage_conf) print("- Overridden log level with: " + str(log_level)) print("- Overridden debug with: " + str(debug)) print("- Overridden trace with: " + str(trace)) if debug: # Add environment variable to get binding-commons debug information os.environ['COMPSS_BINDINGS_DEBUG'] = '1' if trace is False: trace = 0 elif trace == 'basic' or trace is True: trace = 1 os.environ['LD_PRELOAD'] = extrae_lib + '/libpttrace.so' elif trace == 'advanced': trace = 2 os.environ['LD_PRELOAD'] = extrae_lib + '/libpttrace.so' else: print( "ERROR: Wrong tracing parameter ( [ True | basic ] | advanced | False)" ) return -1 # Build a dictionary with all variables needed for initializing the runtime config = dict() config['compss_home'] = compss_home config['debug'] = debug if project_xml is None: project_path = 'Runtime/configuration/xml/projects/default_project.xml' config['project_xml'] = compss_home + os.path.sep + project_path else: config['project_xml'] = project_xml if resources_xml is None: resources_path = 'Runtime/configuration/xml/resources/default_resources.xml' config['resources_xml'] = compss_home + os.path.sep + resources_path else: config['resources_xml'] = resources_xml config['summary'] = summary config['task_execution'] = task_execution config['storage_conf'] = storage_conf config['task_count'] = task_count if app_name is None: config['app_name'] = 'Interactive' else: config['app_name'] = app_name config['uuid'] = uuid config['base_log_dir'] = base_log_dir config['specific_log_dir'] = specific_log_dir config['graph'] = graph config['monitor'] = monitor config['trace'] = trace config['extrae_cfg'] = extrae_cfg config['comm'] = comm config['conn'] = conn config['master_name'] = master_name config['master_port'] = master_port config['scheduler'] = scheduler config['cp'] = cp config['pythonpath'] = pythonpath config['classpath'] = classpath config['ld_library_path'] = ld_library_path config['jvm_workers'] = jvm_workers config['cpu_affinity'] = cpu_affinity config['gpu_affinity'] = gpu_affinity config['fpga_affinity'] = fpga_affinity config['fpga_reprogram'] = fpga_reprogram config['profile_input'] = profile_input config['profile_output'] = profile_output config['scheduler_config'] = scheduler_config if external_adaptation: config['external_adaptation'] = 'true' else: config['external_adaptation'] = 'false' major_version = sys.version_info[0] python_interpreter = 'python' + str(major_version) config['python_interpreter'] = python_interpreter config['python_version'] = str(major_version) if 'VIRTUAL_ENV' in os.environ: # Running within a virtual environment python_virtual_environment = os.environ['VIRTUAL_ENV'] else: python_virtual_environment = 'null' config['python_virtual_environment'] = python_virtual_environment config[ 'python_propagate_virtual_environment'] = propagate_virtual_environment initialize_compss(config) ############################################################## # RUNTIME START ############################################################## print("* - Starting COMPSs runtime... *") sys.stdout.flush() # Force flush compss_start() if o_c is True: # set cross-module variable binding.object_conversion = True else: # set cross-module variable binding.object_conversion = False # Remove launch.py, log_level and object_conversion from sys.argv, # It will be inherited by the app through execfile # sys.argv = sys.argv[3:] # Get application execution path # app_path = sys.argv[0] # not needed in interactive mode global log_path log_path = get_log_path() binding.temp_dir = mkdtemp(prefix='pycompss', dir=log_path + '/tmpFiles/') print("* - Log path : " + log_path) # Logging setup - messages before this step are ignored (need log_path to configure the logger). if debug or log_level == "debug": json_path = '/Bindings/python/' + str( major_version) + '/log/logging.json.debug' init_logging(os.getenv('COMPSS_HOME') + json_path, log_path) elif log_level == "info": json_path = '/Bindings/python/' + str( major_version) + '/log/logging.json.off' init_logging(os.getenv('COMPSS_HOME') + json_path, log_path) elif log_level == "off": json_path = '/Bindings/python/' + str( major_version) + '/log/logging.json.off' init_logging(os.getenv('COMPSS_HOME') + json_path, log_path) else: # Default json_path = '/Bindings/python/' + str( major_version) + '/log/logging.json' init_logging(os.getenv('COMPSS_HOME') + json_path, log_path) logger = logging.getLogger("pycompss.runtime.launch") __print_setup__(verbose, log_level, o_c, debug, graph, trace, monitor, project_xml, resources_xml, summary, task_execution, storage_conf, pythonpath, classpath, ld_library_path, task_count, app_name, uuid, base_log_dir, specific_log_dir, extrae_cfg, comm, conn, master_name, master_port, scheduler, jvm_workers, cpu_affinity, gpu_affinity, fpga_affinity, fpga_reprogram, profile_input, profile_output, scheduler_config, external_adaptation, python_interpreter, major_version, python_virtual_environment, propagate_virtual_environment) logger.debug("--- START ---") logger.debug("PyCOMPSs Log path: %s" % log_path) if storage_conf is not None: logger.debug("Storage configuration file: %s" % storage_conf) from storage.api import init as init_storage init_storage(config_file_path=storage_conf) global persistent_storage persistent_storage = True # MAIN EXECUTION # let the user write an interactive application print("* - PyCOMPSs Runtime started... Have fun! *") print("******************************************************")
def launch_pycompss_application( app, # type: str func, # type: typing.Optional[str] log_level="off", # type: str o_c=False, # type: bool debug=False, # type: bool graph=False, # type: bool trace=False, # type: bool monitor=-1, # type: int project_xml="", # type: str resources_xml="", # type: str summary=False, # type: bool task_execution="compss", # type: str storage_impl="", # type: str storage_conf="", # type: str streaming_backend="", # type: str streaming_master_name="", # type: str streaming_master_port="", # type: str task_count=50, # type: int app_name="", # type: str uuid="", # type: str base_log_dir="", # type: str specific_log_dir="", # type: str extrae_cfg="", # type: str comm="NIO", # type: str conn=DEFAULT_CONN, # type: str master_name="", # type: str master_port="", # type: str scheduler=DEFAULT_SCHED, # type: str jvm_workers=DEFAULT_JVM_WORKERS, # type: str cpu_affinity="automatic", # type: str gpu_affinity="automatic", # type: str fpga_affinity="automatic", # type: str fpga_reprogram="", # type: str profile_input="", # type: str profile_output="", # type: str scheduler_config="", # type: str external_adaptation=False, # type: bool propagate_virtual_environment=True, # type: bool mpi_worker=False, # type: bool worker_cache=False, # type: typing.Union[bool, str] shutdown_in_node_failure=False, # type: bool io_executors=0, # type: int env_script="", # type: str reuse_on_block=True, # type: bool nested_enabled=False, # type: bool tracing_task_dependencies=False, # type: bool trace_label="", # type: str extrae_cfg_python="", # type: str wcl=0, # type: int cache_profiler=False, # type: bool *args, **kwargs): # NOSONAR # type: (...) -> typing.Any """ Launch PyCOMPSs application from function. :param app: Application path :param func: Function :param log_level: Logging level [ "trace"|"debug"|"info"|"api"|"off" ] (default: "off") :param o_c: Objects to string conversion [ True | False ] (default: False) :param debug: Debug mode [ True | False ] (default: False) (overrides log_level) :param graph: Generate graph [ True | False ] (default: False) :param trace: Generate trace [ True | False | "scorep" | "arm-map" | "arm-ddt"] (default: False) :param monitor: Monitor refresh rate (default: None) :param project_xml: Project xml file path :param resources_xml: Resources xml file path :param summary: Execution summary [ True | False ] (default: False) :param task_execution: Task execution (default: "compss") :param storage_impl: Storage implementation path :param storage_conf: Storage configuration file path :param streaming_backend: Streaming backend (default: None) :param streaming_master_name: Streaming master name (default: None) :param streaming_master_port: Streaming master port (default: None) :param task_count: Task count (default: 50) :param app_name: Application name (default: Interactive_date) :param uuid: UUId :param base_log_dir: Base logging directory :param specific_log_dir: Specific logging directory :param extrae_cfg: Extrae configuration file path :param comm: Communication library (default: NIO) :param conn: Connector (default: DefaultSSHConnector) :param master_name: Master Name (default: "") :param master_port: Master port (default: "") :param scheduler: Scheduler (default: es.bsc.compss.scheduler.loadbalancing.LoadBalancingScheduler) :param jvm_workers: Java VM parameters (default: "-Xms1024m,-Xmx1024m,-Xmn400m") :param cpu_affinity: CPU Core affinity (default: "automatic") :param gpu_affinity: GPU Core affinity (default: "automatic") :param fpga_affinity: FPA Core affinity (default: "automatic") :param fpga_reprogram: FPGA reprogram command (default: "") :param profile_input: Input profile (default: "") :param profile_output: Output profile (default: "") :param scheduler_config: Scheduler configuration (default: "") :param external_adaptation: External adaptation [ True | False ] (default: False) :param propagate_virtual_environment: Propagate virtual environment [ True | False ] (default: False) :param mpi_worker: Use the MPI worker [ True | False ] (default: False) :param worker_cache: Use the worker cache [ True | int(size) | False] (default: False) :param shutdown_in_node_failure: Shutdown in node failure [ True | False] (default: False) :param io_executors: <Integer> Number of IO executors :param env_script: <String> Environment script to be sourced in workers :param reuse_on_block: Reuse on block [ True | False] (default: True) :param nested_enabled: Nested enabled [ True | False] (default: False) :param tracing_task_dependencies: Include task dependencies in trace [ True | False] (default: False) :param trace_label: <String> Add trace label :param extrae_cfg_python: <String> Extrae configuration file for the workers :param wcl: <Integer> Wallclock limit. Stops the runtime if reached. 0 means forever. :param cache_profiler: Use the cache profiler [ True | False] (default: False) :param args: Positional arguments :param kwargs: Named arguments :return: Execution result """ # Check that COMPSs is available if "COMPSS_HOME" not in os.environ: # Do not allow to continue if COMPSS_HOME is not defined raise PyCOMPSsException( "ERROR: COMPSS_HOME is not defined in the environment") # Let the Python binding know we are at master context.set_pycompss_context(context.MASTER) # Then we can import the appropriate start and stop functions from the API from pycompss.api.api import compss_start, compss_stop ############################################################## # INITIALIZATION ############################################################## if debug: log_level = "debug" # Initial dictionary with the user defined parameters all_vars = parameters_to_dict( log_level, debug, o_c, graph, trace, monitor, project_xml, resources_xml, summary, task_execution, storage_impl, storage_conf, streaming_backend, streaming_master_name, streaming_master_port, task_count, app_name, uuid, base_log_dir, specific_log_dir, extrae_cfg, comm, conn, master_name, master_port, scheduler, jvm_workers, cpu_affinity, gpu_affinity, fpga_affinity, fpga_reprogram, profile_input, profile_output, scheduler_config, external_adaptation, propagate_virtual_environment, mpi_worker, worker_cache, shutdown_in_node_failure, io_executors, env_script, reuse_on_block, nested_enabled, tracing_task_dependencies, trace_label, extrae_cfg_python, wcl, cache_profiler) # Save all vars in global current flags so that events.py can restart # the notebook with the same flags export_current_flags(all_vars) # Check the provided flags flags, issues = check_flags(all_vars) if not flags: print_flag_issues(issues) return None # Prepare the environment env_vars = prepare_environment(False, o_c, storage_impl, app, debug, trace, mpi_worker) all_vars.update(env_vars) monitoring_vars = prepare_loglevel_graph_for_monitoring( monitor, graph, debug, log_level) all_vars.update(monitoring_vars) if RUNNING_IN_SUPERCOMPUTER: updated_vars = updated_variables_in_sc() all_vars.update(updated_vars) to_update = prepare_tracing_environment(all_vars["trace"], all_vars["extrae_lib"], all_vars["ld_library_path"]) all_vars["trace"], all_vars["ld_library_path"] = to_update inf_vars = check_infrastructure_variables(all_vars["project_xml"], all_vars["resources_xml"], all_vars["compss_home"], all_vars["app_name"], all_vars["file_name"], all_vars["external_adaptation"]) all_vars.update(inf_vars) create_init_config_file(**all_vars) ############################################################## # RUNTIME START ############################################################## # Runtime start compss_start(log_level, all_vars["trace"], True) # Setup logging binding_log_path = get_log_path() log_path = os.path.join(all_vars["compss_home"], "Bindings", "python", str(all_vars["major_version"]), "log") set_temporary_directory(binding_log_path) logging_cfg_file = get_logging_cfg_file(log_level) init_logging(os.path.join(log_path, logging_cfg_file), binding_log_path) logger = logging.getLogger("pycompss.runtime.launch") logger.debug("--- START ---") logger.debug("PyCOMPSs Log path: %s" % log_path) if storage_impl and storage_conf: logger.debug("Starting storage") persistent_storage = master_init_storage(all_vars["storage_conf"], logger) else: persistent_storage = False logger.debug("Starting streaming") streaming = init_streaming(all_vars["streaming_backend"], all_vars["streaming_master_name"], all_vars["streaming_master_port"]) saved_argv = sys.argv sys.argv = list(args) # Execution: with event_master(APPLICATION_RUNNING_EVENT): if func is None or func == "__main__": if IS_PYTHON3: exec(open(app).read()) else: execfile(app) # type: ignore result = None else: if IS_PYTHON3: from importlib.machinery import SourceFileLoader # noqa imported_module = SourceFileLoader( all_vars["file_name"], app).load_module() # type: ignore else: import imp # noqa imported_module = imp.load_source(all_vars["file_name"], app) # noqa method_to_call = getattr(imported_module, func) try: result = method_to_call(*args, **kwargs) except TypeError: result = method_to_call() # Recover the system arguments sys.argv = saved_argv # Stop streaming if streaming: stop_streaming() # Stop persistent storage if persistent_storage: master_stop_storage(logger) logger.debug("--- END ---") ############################################################## # RUNTIME STOP ############################################################## # Stop runtime compss_stop() clean_log_configs() return result
def compss_main(): # type: () -> None """ PyCOMPSs main function. General call: python $PYCOMPSS_HOME/pycompss/runtime/launch.py $wall_clock $log_level $PyObject_serialize $storage_conf $streaming_backend $streaming_master_name $streaming_master_port $fullAppPath $application_args :return: None """ global APP_PATH global STREAMING global PERSISTENT_STORAGE global LOGGER # Let the Python binding know we are at master context.set_pycompss_context(context.MASTER) # Then we can import the appropriate start and stop functions from the API from pycompss.api.api import compss_start # noqa from pycompss.api.api import compss_stop # noqa from pycompss.api.api import compss_set_wall_clock # noqa # See parse_arguments, defined above # In order to avoid parsing user arguments, we are going to remove user # args from sys.argv user_sys_argv = sys.argv[10:] sys.argv = sys.argv[:10] args = parse_arguments() # We are done, now sys.argv must contain user args only sys.argv = [args.app_path] + user_sys_argv # Get log_level log_level = args.log_level # Setup tracing tracing = int(args.tracing) # Get storage configuration at master storage_conf = args.storage_configuration # Load user imports before starting the runtime (can be avoided if # ENVIRONMENT_VARIABLE_LOAD -- defined in configuration.py -- # is set to false). # Reason: some cases like autoparallel can require to avoid loading. # It is disabled if using storage (with dataClay this can not be done) if preload_user_code() and not use_storage(storage_conf): with context.loading_context(): __load_user_module__(args.app_path, log_level) # Start the runtime compss_start(log_level, tracing, False) # Register @implements core elements (they can not be registered in # __load_user__module__). __register_implementation_core_elements__() # Get application wall clock limit wall_clock = int(args.wall_clock) if wall_clock > 0: compss_set_wall_clock(wall_clock) # Get object_conversion boolean set_object_conversion(args.object_conversion == "true") # Get application execution path APP_PATH = args.app_path # Setup logging binding_log_path = get_log_path() log_path = os.path.join(str(os.getenv("COMPSS_HOME")), "Bindings", "python", str(PYTHON_VERSION), "log") set_temporary_directory(binding_log_path) logging_cfg_file = get_logging_cfg_file(log_level) init_logging(os.path.join(log_path, logging_cfg_file), binding_log_path) LOGGER = logging.getLogger("pycompss.runtime.launch") # Get JVM options # jvm_opts = os.environ["JVM_OPTIONS_FILE"] # from pycompss.util.jvm.parser import convert_to_dict # opts = convert_to_dict(jvm_opts) # storage_conf = opts.get("-Dcompss.storage.conf") exit_code = 0 try: if __debug__: LOGGER.debug("--- START ---") LOGGER.debug("PyCOMPSs Log path: %s" % binding_log_path) # Start persistent storage PERSISTENT_STORAGE = master_init_storage(storage_conf, LOGGER) # Start STREAMING STREAMING = init_streaming(args.streaming_backend, args.streaming_master_name, args.streaming_master_port) # Show module warnings if __debug__: show_optional_module_warnings() # MAIN EXECUTION with event_master(APPLICATION_RUNNING_EVENT): # MAIN EXECUTION if IS_PYTHON3: with open(APP_PATH) as f: exec(compile(f.read(), APP_PATH, "exec"), globals()) else: execfile(APP_PATH, globals()) # type: ignore # End if __debug__: LOGGER.debug("--- END ---") except SystemExit as e: # NOSONAR - reraising would not allow to stop the runtime gracefully. if e.code != 0: print("[ ERROR ]: User program ended with exitcode %s." % e.code) print("\t\tShutting down runtime...") exit_code = e.code except SerializerException: exit_code = 1 # If an object that can not be serialized has been used as a parameter. print("[ ERROR ]: Serialization exception") exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception(exc_type, exc_value, exc_traceback) for line in lines: if APP_PATH in line: print("[ ERROR ]: In: %s", line) exit_code = 1 except COMPSsException as e: # Any other exception occurred print("[ ERROR ]: A COMPSs exception occurred: " + str(e)) traceback.print_exc() exit_code = 0 # COMPSs exception is not considered an error except Exception as e: # Any other exception occurred print("[ ERROR ]: An exception occurred: " + str(e)) traceback.print_exc() exit_code = 1 finally: # Stop runtime stop_all(exit_code) clean_log_configs()
def launch_pycompss_application( app, func, log_level='off', o_c=False, debug=False, graph=False, trace=False, monitor=None, project_xml=None, resources_xml=None, summary=False, task_execution='compss', storage_impl=None, storage_conf=None, streaming_backend=None, streaming_master_name=None, streaming_master_port=None, task_count=50, app_name=None, uuid=None, base_log_dir=None, specific_log_dir=None, extrae_cfg=None, comm='NIO', conn='es.bsc.compss.connectors.DefaultSSHConnector', # noqa master_name='', master_port='', scheduler='es.bsc.compss.scheduler.loadbalancing.LoadBalancingScheduler', # noqa jvm_workers='-Xms1024m,-Xmx1024m,-Xmn400m', cpu_affinity='automatic', gpu_affinity='automatic', fpga_affinity='automatic', fpga_reprogram='', profile_input='', profile_output='', scheduler_config='', external_adaptation=False, propagate_virtual_environment=True, mpi_worker=False, *args, **kwargs): """ Launch PyCOMPSs application from function. :param app: Application path :param func: Function :param args: Arguments :param kwargs: Keyword arguments :param log_level: Logging level [ 'off' | 'info' | 'debug' ] (default: 'off') :param o_c: Objects to string conversion [ True | False ] (default: False) :param debug: Debug mode [ True | False ] (default: False) (overrides log_level) :param graph: Generate graph [ True | False ] (default: False) :param trace: Generate trace [ True | False | 'scorep' | 'arm-map' | 'arm-ddt'] (default: False) :param monitor: Monitor refresh rate (default: None) :param project_xml: Project xml file path :param resources_xml: Resources xml file path :param summary: Execution summary [ True | False ] (default: False) :param task_execution: Task execution (default: 'compss') :param storage_impl: Storage implementation path :param storage_conf: Storage configuration file path :param streaming_backend: Streaming backend (default: None) :param streaming_master_name: Streaming master name (default: None) :param streaming_master_port: Streaming master port (default: None) :param task_count: Task count (default: 50) :param app_name: Application name (default: Interactive_date) :param uuid: UUId :param base_log_dir: Base logging directory :param specific_log_dir: Specific logging directory :param extrae_cfg: Extrae configuration file path :param comm: Communication library (default: NIO) :param conn: Connector (default: DefaultSSHConnector) :param master_name: Master Name (default: '') :param master_port: Master port (default: '') :param scheduler: Scheduler (default: es.bsc.compss.scheduler.loadbalancing.LoadBalancingScheduler) :param jvm_workers: Java VM parameters (default: '-Xms1024m,-Xmx1024m,-Xmn400m') :param cpu_affinity: CPU Core affinity (default: 'automatic') :param gpu_affinity: GPU Core affinity (default: 'automatic') :param fpga_affinity: FPA Core affinity (default: 'automatic') :param fpga_reprogram: FPGA repogram command (default: '') :param profile_input: Input profile (default: '') :param profile_output: Output profile (default: '') :param scheduler_config: Scheduler configuration (default: '') :param external_adaptation: External adaptation [ True | False ] (default: False) :param propagate_virtual_environment: Propagate virtual environment [ True | False ] (default: False) :param mpi_worker: Use the MPI worker [ True | False ] (default: False) :param args: Positional arguments :param kwargs: Named arguments :return: Execution result """ # Let the Python binding know we are at master context.set_pycompss_context(context.MASTER) # Then we can import the appropriate start and stop functions from the API from pycompss.api.api import compss_start, compss_stop ############################################################## # INITIALIZATION ############################################################## # TODO: Check that input values are valid # Initial dictionary with the user defined parameters all_vars = { 'log_level': log_level, 'debug': debug, 'o_c': o_c, 'graph': graph, 'trace': trace, 'monitor': monitor, 'project_xml': project_xml, 'resources_xml': resources_xml, 'summary': summary, 'task_execution': task_execution, 'storage_impl': storage_impl, 'storage_conf': storage_conf, 'streaming_backend': streaming_backend, 'streaming_master_name': streaming_master_name, 'streaming_master_port': streaming_master_port, 'task_count': task_count, 'app_name': app_name, 'uuid': uuid, 'base_log_dir': base_log_dir, 'specific_log_dir': specific_log_dir, 'extrae_cfg': extrae_cfg, 'comm': comm, 'conn': conn, 'master_name': master_name, 'master_port': master_port, 'scheduler': scheduler, 'jvm_workers': jvm_workers, 'cpu_affinity': cpu_affinity, 'gpu_affinity': gpu_affinity, 'fpga_affinity': fpga_affinity, 'fpga_reprogram': fpga_reprogram, 'profile_input': profile_input, 'profile_output': profile_output, 'scheduler_config': scheduler_config, 'external_adaptation': external_adaptation, 'propagate_virtual_environment': propagate_virtual_environment, 'mpi_worker': mpi_worker } # Prepare the environment env_vars = prepare_environment(True, o_c, storage_impl, app, debug, trace, mpi_worker) all_vars.update(env_vars) monitoring_vars = prepare_loglevel_graph_for_monitoring( monitor, graph, debug, log_level) all_vars.update(monitoring_vars) if RUNNING_IN_SUPERCOMPUTER: updated_vars = updated_variables_in_sc() all_vars.update(updated_vars) to_update = prepare_tracing_environment(all_vars['trace'], all_vars['extrae_lib'], all_vars['ld_library_path']) all_vars['trace'], all_vars['ld_library_path'] = to_update inf_vars = check_infrastructure_variables(all_vars['project_xml'], all_vars['resources_xml'], all_vars['compss_home'], all_vars['app_name'], all_vars['file_name'], all_vars['external_adaptation']) all_vars.update(inf_vars) create_init_config_file(**all_vars) ############################################################## # RUNTIME START ############################################################## # Runtime start compss_start() # Configure logging log_path = get_log_path() major_version = all_vars['major_version'] compss_home = all_vars['compss_home'] logger = setup_logger(debug, log_level, major_version, compss_home, log_path) logger.debug('--- START ---') logger.debug('PyCOMPSs Log path: %s' % log_path) logger.debug("[LOG] Starting storage") persistent_storage = init_storage(all_vars['storage_conf'], logger) logger.debug("[LOG] Starting streaming") streaming = init_streaming(all_vars['streaming_backend'], all_vars['streaming_master_name'], all_vars['streaming_master_port']) saved_argv = sys.argv sys.argv = args # Execution: if func is None or func == '__main__': execfile(app) result = None else: import imp imported_module = imp.load_source(all_vars['file_name'], app) method_to_call = getattr(imported_module, func) result = method_to_call(*args, **kwargs) # Recover the system arguments sys.argv = saved_argv if persistent_storage: logger.debug("[LOG] Stopping persistent storage") stop_storage() if streaming: logger.debug("[LOG] Stopping streaming") stop_streaming() logger.debug('--- END ---') ############################################################## # RUNTIME STOP ############################################################## compss_stop() return result