def launch_pycompss_application(
app,
func,
log_level='off',
o_c=False,
debug=False,
graph=False,
trace=False,
monitor=None,
project_xml=None,
resources_xml=None,
summary=False,
task_execution='compss',
storage_impl=None,
storage_conf=None,
streaming_backend=None,
streaming_master_name=None,
streaming_master_port=None,
task_count=50,
app_name=None,
uuid=None,
base_log_dir=None,
specific_log_dir=None,
extrae_cfg=None,
comm='NIO',
conn='es.bsc.compss.connectors.DefaultSSHConnector', # noqa: E501
master_name='',
master_port='',
scheduler='es.bsc.compss.scheduler.loadbalancing.LoadBalancingScheduler', # noqa: E501
jvm_workers='-Xms1024m,-Xmx1024m,-Xmn400m',
cpu_affinity='automatic',
gpu_affinity='automatic',
fpga_affinity='automatic',
fpga_reprogram='',
profile_input='',
profile_output='',
scheduler_config='',
external_adaptation=False,
propagate_virtual_environment=True,
mpi_worker=False,
*args,
**kwargs):
"""
Launch PyCOMPSs application from function.
:param app: Application path
:param func: Function
:param args: Arguments
:param kwargs: Keyword arguments
:param log_level: Logging level [ 'off' | 'info' | 'debug' ]
(default: 'off')
:param o_c: Objects to string conversion [ True | False ] (default: False)
:param debug: Debug mode [ True | False ] (default: False)
(overrides log_level)
:param graph: Generate graph [ True | False ] (default: False)
:param trace: Generate trace
[ True | False | 'scorep' | 'arm-map' | 'arm-ddt']
(default: False)
:param monitor: Monitor refresh rate (default: None)
:param project_xml: Project xml file path
:param resources_xml: Resources xml file path
:param summary: Execution summary [ True | False ] (default: False)
:param task_execution: Task execution (default: 'compss')
:param storage_impl: Storage implementation path
:param storage_conf: Storage configuration file path
:param streaming_backend: Streaming backend (default: None)
:param streaming_master_name: Streaming master name (default: None)
:param streaming_master_port: Streaming master port (default: None)
:param task_count: Task count (default: 50)
:param app_name: Application name (default: Interactive_date)
:param uuid: UUId
:param base_log_dir: Base logging directory
:param specific_log_dir: Specific logging directory
:param extrae_cfg: Extrae configuration file path
:param comm: Communication library (default: NIO)
:param conn: Connector (default: DefaultSSHConnector)
:param master_name: Master Name (default: '')
:param master_port: Master port (default: '')
:param scheduler: Scheduler (default:
es.bsc.compss.scheduler.loadbalancing.LoadBalancingScheduler)
:param jvm_workers: Java VM parameters
(default: '-Xms1024m,-Xmx1024m,-Xmn400m')
:param cpu_affinity: CPU Core affinity (default: 'automatic')
:param gpu_affinity: GPU Core affinity (default: 'automatic')
:param fpga_affinity: FPA Core affinity (default: 'automatic')
:param fpga_reprogram: FPGA repogram command (default: '')
:param profile_input: Input profile (default: '')
:param profile_output: Output profile (default: '')
:param scheduler_config: Scheduler configuration (default: '')
:param external_adaptation: External adaptation [ True | False ]
(default: False)
:param propagate_virtual_environment: Propagate virtual environment
[ True | False ] (default: False)
:param mpi_worker: Use the MPI worker [ True | False ] (default: False)
:param args: Positional arguments
:param kwargs: Named arguments
:return: Execution result
"""
# Let the Python binding know we are at master
context.set_pycompss_context(context.MASTER)
# Then we can import the appropriate start and stop functions from the API
from pycompss.api.api import compss_start, compss_stop
##############################################################
# INITIALIZATION
##############################################################
# TODO: Check that input values are valid
# Initial dictionary with the user defined parameters
all_vars = {
'log_level': log_level,
'debug': debug,
'o_c': o_c,
'graph': graph,
'trace': trace,
'monitor': monitor,
'project_xml': project_xml,
'resources_xml': resources_xml,
'summary': summary,
'task_execution': task_execution,
'storage_impl': storage_impl,
'storage_conf': storage_conf,
'streaming_backend': streaming_backend,
'streaming_master_name': streaming_master_name,
'streaming_master_port': streaming_master_port,
'task_count': task_count,
'app_name': app_name,
'uuid': uuid,
'base_log_dir': base_log_dir,
'specific_log_dir': specific_log_dir,
'extrae_cfg': extrae_cfg,
'comm': comm,
'conn': conn,
'master_name': master_name,
'master_port': master_port,
'scheduler': scheduler,
'jvm_workers': jvm_workers,
'cpu_affinity': cpu_affinity,
'gpu_affinity': gpu_affinity,
'fpga_affinity': fpga_affinity,
'fpga_reprogram': fpga_reprogram,
'profile_input': profile_input,
'profile_output': profile_output,
'scheduler_config': scheduler_config,
'external_adaptation': external_adaptation,
'propagate_virtual_environment': propagate_virtual_environment,
'mpi_worker': mpi_worker
}
# Prepare the environment
env_vars = prepare_environment(True, o_c, storage_impl, app, debug, trace,
mpi_worker)
all_vars.update(env_vars)
monitoring_vars = prepare_loglevel_graph_for_monitoring(
monitor, graph, debug, log_level)
all_vars.update(monitoring_vars)
if RUNNING_IN_SUPERCOMPUTER:
updated_vars = updated_variables_in_sc()
all_vars.update(updated_vars)
to_update = prepare_tracing_environment(all_vars['trace'],
all_vars['extrae_lib'],
all_vars['ld_library_path'])
all_vars['trace'], all_vars['ld_library_path'] = to_update
inf_vars = check_infrastructure_variables(all_vars['project_xml'],
all_vars['resources_xml'],
all_vars['compss_home'],
all_vars['app_name'],
all_vars['file_name'],
all_vars['external_adaptation'])
all_vars.update(inf_vars)
create_init_config_file(**all_vars)
##############################################################
# RUNTIME START
##############################################################
# Runtime start
compss_start()
# Configure logging
log_path = get_log_path()
major_version = all_vars['major_version']
compss_home = all_vars['compss_home']
logger = setup_logger(debug, log_level, major_version, compss_home,
log_path)
logger.debug('--- START ---')
logger.debug('PyCOMPSs Log path: %s' % log_path)
logger.debug("[LOG] Starting storage")
persistent_storage = init_storage(all_vars['storage_conf'], logger)
logger.debug("[LOG] Starting streaming")
streaming = init_streaming(all_vars['streaming_backend'],
all_vars['streaming_master_name'],
all_vars['streaming_master_port'], logger)
saved_argv = sys.argv
sys.argv = args
# Execution:
if func is None or func == '__main__':
if IS_PYTHON3:
exec(open(app).read())
else:
execfile(app) # noqa
result = None
else:
if IS_PYTHON3:
import importlib.util
spec = importlib.util.spec_from_file_location(
all_vars['file_name'], app) # noqa: E501
imported_module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(imported_module)
else:
import imp # noqa
imported_module = imp.load_source(all_vars['file_name'],
app) # noqa
method_to_call = getattr(imported_module, func)
result = method_to_call(*args, **kwargs)
# Recover the system arguments
sys.argv = saved_argv
if persistent_storage:
logger.debug("[LOG] Stopping persistent storage")
stop_storage()
if streaming:
logger.debug("[LOG] Stopping streaming")
stop_streaming(logger)
logger.debug('--- END ---')
##############################################################
# RUNTIME STOP
##############################################################
compss_stop()
return result
def start(log_level='off',
debug=False,
o_c=False,
graph=False,
trace=False,
monitor=None,
project_xml=None,
resources_xml=None,
summary=False,
task_execution='compss',
storage_impl=None,
storage_conf=None,
streaming_backend=None,
streaming_master_name=None,
streaming_master_port=None,
task_count=50,
app_name='Interactive',
uuid=None,
base_log_dir=None,
specific_log_dir=None,
extrae_cfg=None,
comm='NIO',
conn='es.bsc.compss.connectors.DefaultSSHConnector',
master_name='',
master_port='',
scheduler='es.bsc.compss.scheduler.' +
'loadbalancing.LoadBalancingScheduler',
jvm_workers='-Xms1024m,-Xmx1024m,-Xmn400m',
cpu_affinity='automatic',
gpu_affinity='automatic',
fpga_affinity='automatic',
fpga_reprogram='',
profile_input='',
profile_output='',
scheduler_config='',
external_adaptation=False,
propagate_virtual_environment=True,
mpi_worker=False,
verbose=False):
"""
Start the runtime in interactive mode.
:param log_level: Logging level [ 'off' | 'info' | 'debug' ]
(default: 'off')
:param debug: Debug mode [ True | False ]
(default: False) (overrides log-level)
:param o_c: Objects to string conversion [ True | False ]
(default: False)
:param graph: Generate graph [ True | False ]
(default: False)
:param trace: Generate trace
[ True | False | 'scorep' | 'arm-map' | 'arm-ddt' ]
(default: False)
:param monitor: Monitor refresh rate
(default: None)
:param project_xml: Project xml file path
(default: None)
:param resources_xml: Resources xml file path
(default: None)
:param summary: Execution summary [ True | False ]
(default: False)
:param task_execution: Task execution
(default: 'compss')
:param storage_impl: Storage implementation path
(default: None)
:param storage_conf: Storage configuration file path
(default: None)
:param streaming_backend: Streaming backend
(default: None)
:param streaming_master_name: Streaming master name
(default: None)
:param streaming_master_port: Streaming master port
(default: None)
:param task_count: Task count
(default: 50)
:param app_name: Application name
default: Interactive_date)
:param uuid: UUId
(default: None)
:param base_log_dir: Base logging directory
(default: None)
:param specific_log_dir: Specific logging directory
(default: None)
:param extrae_cfg: Extrae configuration file path
(default: None)
:param comm: Communication library
(default: NIO)
:param conn: Connector
(default: DefaultSSHConnector)
:param master_name: Master Name
(default: '')
:param master_port: Master port
(default: '')
:param scheduler: Scheduler
(default: es.bsc.compss.scheduler.
loadbalancing.LoadBalancingScheduler)
:param jvm_workers: Java VM parameters
(default: '-Xms1024m,-Xmx1024m,-Xmn400m')
:param cpu_affinity: CPU Core affinity
(default: 'automatic')
:param gpu_affinity: GPU affinity
(default: 'automatic')
:param fpga_affinity: FPGA affinity
(default: 'automatic')
:param fpga_reprogram: FPGA repogram command
(default: '')
:param profile_input: Input profile
(default: '')
:param profile_output: Output profile
(default: '')
:param scheduler_config: Scheduler configuration
(default: '')
:param external_adaptation: External adaptation [ True | False ]
(default: False)
:param propagate_virtual_environment: Propagate virtual environment
[ True | False ]
(default: False)
:param mpi_worker: Use the MPI worker [ True | False ]
(default: False)
:param verbose: Verbose mode [ True | False ]
(default: False)
:return: None
"""
# Export global variables
global GRAPHING
GRAPHING = graph
__export_globals__()
interactive_helpers.DEBUG = debug
__show_flower__()
# Let the Python binding know we are at master
context.set_pycompss_context(context.MASTER)
# Then we can import the appropriate start and stop functions from the API
from pycompss.api.api import compss_start
##############################################################
# INITIALIZATION
##############################################################
# TODO: Check that input values are valid
# Initial dictionary with the user defined parameters
all_vars = {
'log_level': log_level,
'debug': debug,
'o_c': o_c,
'graph': graph,
'trace': trace,
'monitor': monitor,
'project_xml': project_xml,
'resources_xml': resources_xml,
'summary': summary,
'task_execution': task_execution,
'storage_impl': storage_impl,
'storage_conf': storage_conf,
'streaming_backend': streaming_backend,
'streaming_master_name': streaming_master_name,
'streaming_master_port': streaming_master_port,
'task_count': task_count,
'app_name': app_name,
'uuid': uuid,
'base_log_dir': base_log_dir,
'specific_log_dir': specific_log_dir,
'extrae_cfg': extrae_cfg,
'comm': comm,
'conn': conn,
'master_name': master_name,
'master_port': master_port,
'scheduler': scheduler,
'jvm_workers': jvm_workers,
'cpu_affinity': cpu_affinity,
'gpu_affinity': gpu_affinity,
'fpga_affinity': fpga_affinity,
'fpga_reprogram': fpga_reprogram,
'profile_input': profile_input,
'profile_output': profile_output,
'scheduler_config': scheduler_config,
'external_adaptation': external_adaptation,
'propagate_virtual_environment': propagate_virtual_environment,
'mpi_worker': mpi_worker
}
# Prepare the environment
env_vars = prepare_environment(True, o_c, storage_impl, None, debug, trace,
mpi_worker)
all_vars.update(env_vars)
# Update the log level and graph values if monitoring is enabled
monitoring_vars = prepare_loglevel_graph_for_monitoring(
monitor, graph, debug, log_level)
all_vars.update(monitoring_vars)
# Check if running in supercomputer and update the variables accordingly
# with the defined in the launcher and exported in environment variables.
if RUNNING_IN_SUPERCOMPUTER:
updated_vars = updated_variables_in_sc()
if verbose:
print("- Overridden project xml with: " +
updated_vars['project_xml'])
print("- Overridden resources xml with: " +
updated_vars['resources_xml'])
print("- Overridden master name with: " +
updated_vars['master_name'])
print("- Overridden master port with: " +
updated_vars['master_port'])
print("- Overridden uuid with: " + updated_vars['uuid'])
print("- Overridden base log dir with: " +
updated_vars['base_log_dir'])
print("- Overridden specific log dir with: " +
updated_vars['specific_log_dir'])
print("- Overridden storage conf with: " +
updated_vars['storage_conf'])
print("- Overridden log level with: " +
str(updated_vars['log_level']))
print("- Overridden debug with: " + str(updated_vars['debug']))
print("- Overridden trace with: " + str(updated_vars['trace']))
all_vars.update(updated_vars)
# Update the tracing environment if set and set the appropriate trace
# integer value
tracing_vars = prepare_tracing_environment(all_vars['trace'],
all_vars['extrae_lib'],
all_vars['ld_library_path'])
all_vars['trace'], all_vars['ld_library_path'] = tracing_vars
# Update the infrastructure variables if necessary
inf_vars = check_infrastructure_variables(all_vars['project_xml'],
all_vars['resources_xml'],
all_vars['compss_home'],
all_vars['app_name'],
all_vars['file_name'],
all_vars['external_adaptation'])
all_vars.update(inf_vars)
# With all this information, create the configuration file for the
# runtime start
create_init_config_file(**all_vars)
##############################################################
# RUNTIME START
##############################################################
print("* - Starting COMPSs runtime... *")
sys.stdout.flush() # Force flush
compss_start()
global LOG_PATH
LOG_PATH = get_log_path()
binding.temp_dir = mkdtemp(prefix='pycompss', dir=LOG_PATH + '/tmpFiles/')
print("* - Log path : " + LOG_PATH)
major_version = all_vars['major_version']
compss_home = all_vars['compss_home']
logger = setup_logger(debug, log_level, major_version, compss_home,
LOG_PATH)
__print_setup__(verbose, all_vars)
logger.debug("--- START ---")
logger.debug("PyCOMPSs Log path: %s" % LOG_PATH)
logger.debug("Starting storage")
global PERSISTENT_STORAGE
PERSISTENT_STORAGE = init_storage(all_vars['storage_conf'], logger)
logger.debug("Starting streaming")
global STREAMING
STREAMING = init_streaming(all_vars['streaming_backend'],
all_vars['streaming_master_name'],
all_vars['streaming_master_port'], logger)
# MAIN EXECUTION
# let the user write an interactive application
print("* - PyCOMPSs Runtime started... Have fun! *")
print("******************************************************")
def compss_main():
"""
General call:
python $PYCOMPSS_HOME/pycompss/runtime/launch.py $log_level
$PyObject_serialize $storage_conf $streaming_backend
$streaming_master_name $streaming_master_port
$fullAppPath $application_args
:return: None
"""
global APP_PATH
# Let the Python binding know we are at master
context.set_pycompss_context(context.MASTER)
# Then we can import the appropriate start and stop functions from the API
from pycompss.api.api import compss_start, compss_stop
# Start the runtime, see bindings commons
compss_start()
# See parse_arguments, defined above
# In order to avoid parsing user arguments, we are going to remove user
# args from sys.argv
user_sys_argv = sys.argv[8:]
sys.argv = sys.argv[:8]
args = parse_arguments()
# We are done, now sys.argv must contain user args only
sys.argv = [args.app_path] + user_sys_argv
# Get log_level
log_level = args.log_level
# Get object_conversion boolean
binding.object_conversion = args.object_conversion == 'true'
# Get storage configuration at master
storage_conf = args.storage_configuration
# Get application execution path
APP_PATH = args.app_path
binding_log_path = get_log_path()
log_path = os.path.join(os.getenv('COMPSS_HOME'), 'Bindings', 'python',
str(_py_version), 'log')
binding.temp_dir = mkdtemp(prefix='pycompss',
dir=os.path.join(binding_log_path, 'tmpFiles/'))
logging_cfg_file = get_logging_cfg_file(log_level)
init_logging(os.path.join(log_path, logging_cfg_file), binding_log_path)
logger = None
if __debug__:
logger = logging.getLogger("pycompss.runtime.launch")
# Get JVM options
# jvm_opts = os.environ['JVM_OPTIONS_FILE']
# from pycompss.util.jvm.parser import convert_to_dict
# opts = convert_to_dict(jvm_opts)
# storage_conf = opts.get('-Dcompss.storage.conf')
exit_code = 0
try:
if __debug__:
logger.debug('--- START ---')
logger.debug('PyCOMPSs Log path: %s' % binding_log_path)
# Start persistent storage
if __debug__:
logger.debug("[LOG] Starting storage")
persistent_storage = init_storage(storage_conf, logger)
# Start streaming
if __debug__:
logger.debug("[LOG] Starting streaming")
streaming = init_streaming(args.streaming_backend,
args.streaming_master_name,
args.streaming_master_port, logger)
# Show module warnings
if __debug__:
show_optional_module_warnings()
# MAIN EXECUTION
if IS_PYTHON3:
with open(APP_PATH) as f:
exec(compile(f.read(), APP_PATH, 'exec'), globals())
else:
execfile(APP_PATH, globals()) # MAIN EXECUTION
# Stop streaming
if __debug__:
logger.debug("[LOG] Stopping streaming")
if streaming:
stop_streaming(logger)
# Stop persistent storage
if __debug__:
logger.debug("[LOG] Stopping storage")
if persistent_storage:
stop_storage()
# End
if __debug__:
logger.debug('--- END ---')
except SystemExit as e:
if e.code != 0:
print('[ ERROR ]: User program ended with exitcode %s.' % e.code)
print('\t\tShutting down runtime...')
exit_code = e.code
except SerializerException:
exit_code = 1
# If an object that can not be serialized has been used as a parameter.
print("[ ERROR ]: Serialization exception")
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
for line in lines:
if APP_PATH in line:
print('[ ERROR ]: In: %s', line)
exit_code = 1
except COMPSsException as e:
# Any other exception occurred
print("[ ERROR ]: A COMPSs exception occurred: " + str(e))
traceback.print_exc()
exit_code = 0 # COMPSs exception is not considered an error
except Exception as e:
# Any other exception occurred
print("[ ERROR ]: An exception occurred: " + str(e))
traceback.print_exc()
exit_code = 1
finally:
compss_stop(exit_code)
sys.stdout.flush()
sys.stderr.flush()
sys.exit(exit_code)