def compss_main():
'''
General call:
python $PYCOMPSS_HOME/pycompss/runtime/launch.py $log_level $PyObject_serialize $storageConf $fullAppPath $application_args
'''
global app_path
# Start the runtime, see bindings commons
compss_start()
# See parse_arguments, defined above
# In order to avoid parsing user arguments, we are going to remove user
# args from sys.argv
user_sys_argv = sys.argv[5:]
sys.argv = sys.argv[:5]
args = parse_arguments()
# We are done, now sys.argv must contain user args only
sys.argv = [args.app_path] + user_sys_argv
# Get log_level
log_level = args.log_level
# Get object_conversion boolean
binding.object_conversion = args.object_conversion == 'true'
# Get storage configuration at master
storage_conf = args.storage_configuration
persistent_storage = False
if storage_conf != 'null':
persistent_storage = True
from storage.api import init as init_storage
from storage.api import finish as finish_storage
# Get application execution path
app_path = args.app_path
binding_log_path = get_log_path()
log_path = os.path.join(os.getenv('COMPSS_HOME'), 'Bindings', 'python',
str(py_version), 'log')
binding.temp_dir = mkdtemp(prefix='pycompss',
dir=os.path.join(binding_log_path, 'tmpFiles/'))
logging_cfg_file = get_logging_cfg_file(log_level)
init_logging(os.path.join(log_path, logging_cfg_file), binding_log_path)
if __debug__:
logger = logging.getLogger('pycompss.runtime.launch')
# Get JVM options
jvm_opts = os.environ['JVM_OPTIONS_FILE']
opts = convert_to_dict(jvm_opts)
# storage_conf = opts.get('-Dcompss.storage.conf')
try:
if __debug__:
logger.debug('--- START ---')
logger.debug('PyCOMPSs Log path: %s' % binding_log_path)
if persistent_storage:
if __debug__:
logger.debug('Storage configuration file: %s' % storage_conf)
init_storage(config_file_path=storage_conf)
show_optional_module_warnings()
# MAIN EXECUTION
if sys.version_info >= (3, 0):
exec(compile(open(app_path).read(), app_path, 'exec'), globals())
else:
execfile(app_path, globals()) # MAIN EXECUTION
if persistent_storage:
finish_storage()
if __debug__:
logger.debug('--- END ---')
except SystemExit as e:
if e.code != 0: # Seems this is not happening
print('[ ERROR ]: User program ended with exitcode %s.' % e.code)
print('\t\tShutting down runtime...')
except SerializerException:
# If an object that can not be serialized has been used as a parameter.
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
for line in lines:
if app_path in line:
print('[ ERROR ]: In: %s', line)
finally:
compss_stop()
sys.stdout.flush()
sys.stderr.flush()
def start(
log_level='off',
o_c=False,
debug=False,
graph=False,
trace=False,
monitor=None,
project_xml=None,
resources_xml=None,
summary=False,
taskExecution='compss',
storageConf=None,
taskCount=50,
appName='Interactive',
uuid=None,
baseLogDir=None,
specificLogDir=None,
extraeCfg=None,
comm='NIO',
conn='es.bsc.compss.connectors.DefaultSSHConnector',
masterName='',
masterPort='',
scheduler='es.bsc.compss.scheduler.loadBalancingScheduler.LoadBalancingScheduler',
jvmWorkers='-Xms1024m,-Xmx1024m,-Xmn400m',
cpuAffinity='automatic',
gpuAffinity='automatic',
profileInput='',
profileOutput='',
scheduler_config='',
external_adaptation=False,
propagate_virtual_environment=False,
verbose=False):
launchPath = os.path.dirname(os.path.realpath(__file__))
# compss_home = launchPath without the last 4 folders:
# Bindings/python/version/pycompss
compss_home = os.path.sep.join(launchPath.split(os.path.sep)[:-4])
os.environ['COMPSS_HOME'] = compss_home
# Get environment variables
cp = os.getcwd() + '/'
pythonPath = os.environ['PYTHONPATH']
classPath = os.environ['CLASSPATH']
ld_library_path = os.environ['LD_LIBRARY_PATH']
# Set extrae dependencies
extrae_home = compss_home + '/Dependencies/extrae'
extrae_lib = extrae_home + '/lib'
os.environ['EXTRAE_HOME'] = extrae_home
os.environ['LD_LIBRARY_PATH'] = extrae_lib + ':' + ld_library_path
if trace is False:
trace = 0
elif trace == 'basic' or trace is True:
trace = 1
os.environ['LD_PRELOAD'] = extrae_lib + '/libpttrace.so'
elif trace == 'advanced':
trace = 2
os.environ['LD_PRELOAD'] = extrae_lib + '/libpttrace.so'
else:
print('ERROR: Wrong tracing parameter ( [ True | basic ] | \
advanced | False)')
return -1
if monitor is not None:
# Enable the graph if the monitoring is enabled
graph = True
global graphing
graphing = graph
global tracing
tracing = trace
global monitoring
monitoring = monitor
__export_globals__()
print("********************************************************")
print("*************** PyCOMPSs Interactive *******************")
print("********************************************************")
print("* .-~~-.--. ____ ____ *")
print("* : ) |___ \ |___ \ *")
print("* .~ ~ -.\ /.- ~~ . __) | __) | *")
print("* > `. .' < / __/ _ / __/ *")
print("* ( .- -. ) |_____| |_| |_____| *")
print("* `- -.-~ `- -' ~-.- -' *")
print("* ( : ) _ _ .-: *")
print("* ~--. : .--~ .-~ .-~ } *")
print("* ~-.-^-.-~ \_ .~ .-~ .~ *")
print("* \ \ ' \ '_ _ -~ *")
print("* \`.\`. // *")
print("* . - ~ ~-.__\`.\`-.// *")
print("* .-~ . - ~ }~ ~ ~-.~-. *")
print("* .' .-~ .-~ :/~-.~-./: *")
print("* /_~_ _ . - ~ ~-.~-._ *")
print("* ~-.< *")
print("********************************************************")
# print("* .-~~-.--. ____ _____ *")
# print("* : ) |___ \ |___ / *")
# print("* .~ ~ -.\ /.- ~~ . __) | |_ \ *")
# print("* > `. .' < / __/ _ __) | *")
# print("* ( .- -. ) |_____| |_| |___/ *")
##############################################################
# INITIALIZATION
##############################################################
# Build a dictionary with all variables needed for initializing the runtime
config = {}
config['compss_home'] = compss_home
config['debug'] = debug
if project_xml is None:
projPath = 'Runtime/configuration/xml/projects/default_project.xml'
config['project_xml'] = compss_home + os.path.sep + projPath
else:
config['project_xml'] = project_xml
if resources_xml is None:
resPath = 'Runtime/configuration/xml/resources/default_resources.xml'
config['resources_xml'] = compss_home + os.path.sep + resPath
else:
config['resources_xml'] = resources_xml
config['summary'] = summary
config['taskExecution'] = taskExecution
config['storageConf'] = storageConf
config['taskCount'] = taskCount
if appName is None:
config['appName'] = 'Interactive'
else:
config['appName'] = appName
config['uuid'] = uuid
config['baseLogDir'] = baseLogDir
config['specificLogDir'] = specificLogDir
config['graph'] = graph
config['monitor'] = monitor
config['trace'] = trace
config['extraeCfg'] = extraeCfg
config['comm'] = comm
config['conn'] = conn
config['masterName'] = masterName
config['masterPort'] = masterPort
config['scheduler'] = scheduler
config['cp'] = cp
config['classpath'] = classPath
config['jvmWorkers'] = jvmWorkers
config['pythonPath'] = pythonPath
config['cpuAffinity'] = cpuAffinity
config['gpuAffinity'] = gpuAffinity
config['profileInput'] = profileInput
config['profileOutput'] = profileOutput
config['scheduler_config'] = scheduler_config
if external_adaptation:
config['external_adaptation'] = 'true'
else:
config['external_adaptation'] = 'false'
major_version = sys.version_info[0]
python_interpreter = 'python' + str(major_version)
config['python_interpreter'] = python_interpreter
config['python_version'] = str(major_version)
if 'VIRTUAL_ENV' in os.environ:
# Running within a virtual environment
python_virtual_environment = os.environ['VIRTUAL_ENV']
else:
python_virtual_environment = 'null'
config['python_virtual_environment'] = python_virtual_environment
config[
'python_propagate_virtual_environment'] = propagate_virtual_environment
initialize_compss(config)
##############################################################
# RUNTIME START
##############################################################
print("* - Starting COMPSs runtime... *")
compss_start()
if o_c is True:
# set cross-module variable
binding.object_conversion = True
else:
# set cross-module variable
binding.object_conversion = False
# Remove launch.py, log_level and object_conversion from sys.argv,
# It will be inherited by the app through execfile
# sys.argv = sys.argv[3:]
# Get application execution path
# app_path = sys.argv[0] # not needed in interactive mode
global log_path
log_path = get_log_path()
binding.temp_dir = mkdtemp(prefix='pycompss', dir=log_path + '/tmpFiles/')
print("* - Log path : " + log_path)
# Logging setup
if log_level == "debug":
jsonPath = '/Bindings/python/' + str(
major_version) + '/log/logging.json.debug'
init_logging(os.getenv('COMPSS_HOME') + jsonPath, log_path)
elif log_level == "info":
jsonPath = '/Bindings/python/' + str(
major_version) + '/log/logging.json.off'
init_logging(os.getenv('COMPSS_HOME') + jsonPath, log_path)
elif log_level == "off":
jsonPath = '/Bindings/python/' + str(
major_version) + '/log/logging.json.off'
init_logging(os.getenv('COMPSS_HOME') + jsonPath, log_path)
else:
# Default
jsonPath = '/Bindings/python/' + str(
major_version) + '/log/logging.json'
init_logging(os.getenv('COMPSS_HOME') + jsonPath, log_path)
logger = logging.getLogger("pycompss.runtime.launch")
printSetup(verbose, log_level, o_c, debug, graph, trace, monitor,
project_xml, resources_xml, summary, taskExecution, storageConf,
taskCount, appName, uuid, baseLogDir, specificLogDir, extraeCfg,
comm, conn, masterName, masterPort, scheduler, jvmWorkers,
cpuAffinity, gpuAffinity, profileInput, profileOutput,
scheduler_config, external_adaptation, python_interpreter,
major_version, python_virtual_environment,
propagate_virtual_environment)
logger.debug("--- START ---")
logger.debug("PyCOMPSs Log path: %s" % log_path)
if storageConf is not None:
logger.debug("Storage configuration file: %s" % storageConf)
from storage.api import init as initStorage
initStorage(config_file_path=storageConf)
global persistent_storage
persistent_storage = True
# MAIN EXECUTION
# let the user write an interactive application
print("* - PyCOMPSs Runtime started... Have fun! *")
print("********************************************************")
def launch_pycompss_application(
app,
func,
args=[],
kwargs={},
log_level='off',
o_c=False,
debug=False,
graph=False,
trace=False,
monitor=None,
project_xml=None,
resources_xml=None,
summary=False,
taskExecution='compss',
storageConf=None,
taskCount=50,
appName=None,
uuid=None,
baseLogDir=None,
specificLogDir=None,
extraeCfg=None,
comm='NIO',
conn='es.bsc.compss.connectors.DefaultSSHConnector',
masterName='',
masterPort='',
scheduler='es.bsc.compss.scheduler.loadBalancingScheduler.LoadBalancingScheduler',
jvmWorkers='-Xms1024m,-Xmx1024m,-Xmn400m',
obj_conv=False,
cpuAffinity='automatic',
gpuAffinity='automatic',
profileInput='',
profileOutput='',
scheduler_config='',
external_adaptation=False,
python_propagate_virtual_environment=True):
global app_path
launchPath = os.path.dirname(os.path.abspath(__file__))
# compss_home = launchPath without the last 4 folders:
# (Bindings/python/pycompss/runtime)
compss_home = os.path.sep.join(launchPath.split(os.path.sep)[:-4])
# Grab the existing PYTHONPATH and CLASSPATH values
pythonpath = os.environ['PYTHONPATH']
classpath = os.environ['CLASSPATH']
# Enable/Disable object to string conversion
binding.object_conversion = obj_conv
# Get the filename and its path.
file_name = os.path.splitext(os.path.basename(app))[0]
cp = os.path.dirname(app)
# Build a dictionary with all variables needed for initializing the runtime.
config = {}
config['compss_home'] = compss_home
config['debug'] = debug
if project_xml is None:
projXml = 'Runtime/configuration/xml/projects/default_project.xml'
config['project_xml'] = compss_home + os.path.sep + projXml
else:
config['project_xml'] = project_xml
if resources_xml is None:
resXml = 'Runtime/configuration/xml/resources/default_resources.xml'
config['resources_xml'] = compss_home + os.path.sep + resXml
else:
config['resources_xml'] = resources_xml
config['summary'] = summary
config['taskExecution'] = taskExecution
config['storageConf'] = storageConf
config['taskCount'] = taskCount
if appName is None:
config['appName'] = file_name
else:
config['appName'] = appName
config['uuid'] = uuid
config['baseLogDir'] = baseLogDir
config['specificLogDir'] = specificLogDir
config['graph'] = graph
config['monitor'] = monitor
config['trace'] = trace
config['extraeCfg'] = extraeCfg
config['comm'] = comm
config['conn'] = conn
config['masterName'] = masterName
config['masterPort'] = masterPort
config['scheduler'] = scheduler
config['cp'] = cp
config['classpath'] = classpath
config['jvmWorkers'] = jvmWorkers
config['pythonPath'] = pythonpath
config['cpuAffinity'] = cpuAffinity
config['gpuAffinity'] = gpuAffinity
config['profileInput'] = profileInput
config['profileOutput'] = profileOutput
config['scheduler_config'] = scheduler_config
if external_adaptation:
config['external_adaptation'] = 'true'
else:
config['external_adaptation'] = 'false'
config['python_interpreter'] = 'python' + str(sys.version_info[0])
config['python_version'] = str(sys.version_info[0])
if 'VIRTUAL_ENV' in os.environ:
# Running within a virtual environment
python_virtual_environment = os.environ['VIRTUAL_ENV']
else:
python_virtual_environment = 'null'
config['python_virtual_environment'] = python_virtual_environment
config[
'python_propagate_virtual_environment'] = python_propagate_virtual_environment
initialize_compss(config)
# Runtime start
compss_start()
# Configure logging
app_path = app
log_path = get_log_path()
if debug:
# DEBUG
init_logging(compss_home + '/Bindings/python/log/logging.json.debug',
log_path)
else:
# NO DEBUG
init_logging(compss_home + '/Bindings/python/log/logging.json',
log_path)
logger = logging.getLogger("pycompss.runtime.launch")
logger.debug('--- START ---')
logger.debug('PyCOMPSs Log path: %s' % log_path)
saved_argv = sys.argv
sys.argv = args
# Execution:
if func is None or func == '__main__':
result = execfile(app)
else:
import imp
imported_module = imp.load_source(file_name, app)
methodToCall = getattr(imported_module, func)
result = methodToCall(*args, **kwargs)
# Recover the system arguments
sys.argv = saved_argv
logger.debug('--- END ---')
compss_stop()
return result
def launch_pycompss_application(
app,
func,
log_level='off',
o_c=False,
debug=False,
graph=False,
trace=False,
monitor=None,
project_xml=None,
resources_xml=None,
summary=False,
task_execution='compss',
storage_impl=None,
storage_conf=None,
task_count=50,
app_name=None,
uuid=None,
base_log_dir=None,
specific_log_dir=None,
extrae_cfg=None,
comm='NIO',
conn='es.bsc.compss.connectors.DefaultSSHConnector',
master_name='',
master_port='',
scheduler='es.bsc.compss.scheduler.loadBalancingScheduler.LoadBalancingScheduler',
jvm_workers='-Xms1024m,-Xmx1024m,-Xmn400m',
obj_conv=False,
cpu_affinity='automatic',
gpu_affinity='automatic',
fpga_affinity='automatic',
fpga_reprogram='',
profile_input='',
profile_output='',
scheduler_config='',
external_adaptation=False,
python_propagate_virtual_environment=True,
*args,
**kwargs):
"""
Launch pycompss application.
:param app: Application path
:param func: Function
:param args: Arguments
:param kwargs: Keyword arguments
:param log_level: Logging level [ 'on' | 'off'] (default: 'off')
:param o_c: Objects to string conversion [ True | False ] (default: False)
:param debug: Debug mode [ True | False ] (default: False)
:param graph: Generate graph [ True | False ] (default: False)
:param trace: Generate trace [ True | False ] (default: False)
:param monitor: Monitor refresh rate (default: None)
:param project_xml: Project xml file path
:param resources_xml: Resources xml file path
:param summary: Execution summary [ True | False ] (default: False)
:param task_execution: Task execution (default: 'compss')
:param storage_impl: Storage implementation path
:param storage_conf: Storage configuration file path
:param task_count: Task count (default: 50)
:param app_name: Application name (default: Interactive_date)
:param uuid: UUId
:param base_log_dir: Base logging directory
:param specific_log_dir: Specific logging directory
:param extrae_cfg: Extrae configuration file path
:param comm: Communication library (default: NIO)
:param conn: Connector (default: DefaultSSHConnector)
:param master_name: Master Name (default: '')
:param master_port: Master port (default: '')
:param scheduler: Scheduler (default: LoadBalancingScheduler)
:param jvm_workers: Java VM parameters (default: '-Xms1024m,-Xmx1024m,-Xmn400m')
:param cpu_affinity: CPU Core affinity (default: 'automatic')
:param gpu_affinity: GPU Core affinity (default: 'automatic')
:param fpga_affinity: FPA Core affinity (default: 'automatic')
:param fpga_reprogram: FPGA repogram command (default: '')
:param profile_input: Input profile (default: '')
:param profile_output: Output profile (default: '')
:param scheduler_config: Scheduler configuration (default: '')
:param external_adaptation: External adaptation [ True | False ] (default: False)
:param python_propagate_virtual_environment: Propagate virtual environment [ True | False ] (default: False)
:return: Execution result
"""
global app_path
# Let the Python binding know we are at master
set_pycompss_context('MASTER')
launch_path = os.path.dirname(os.path.abspath(__file__))
# compss_home = launch_path without the last 4 folders:
# (Bindings/python/pycompss/runtime)
compss_home = os.path.sep.join(launch_path.split(os.path.sep)[:-4])
# Grab the existing PYTHONPATH and CLASSPATH values
pythonpath = os.environ['PYTHONPATH']
classpath = os.environ['CLASSPATH']
ld_library_path = os.environ['LD_LIBRARY_PATH']
# Enable/Disable object to string conversion
binding.object_conversion = obj_conv
# Get the filename and its path.
file_name = os.path.splitext(os.path.basename(app))[0]
cp = os.path.dirname(app)
# Set storage classpath
if storage_impl:
if storage_impl == 'redis':
cp = cp + ':' + compss_home + '/Tools/storage/redis/compss-redisPSCO.jar'
else:
cp = cp + ':' + storage_impl
if RUNNING_IN_SUPERCOMPUTER:
# Since the deployment in supercomputers is done through the use of enqueue_compss
# and consequently launch_compss - the project and resources xmls are already created
project_xml, resources_xml = get_xmls()
# It also exported some environment variables that we need here
master_name = get_master_node()
master_port = get_master_port()
uuid = get_uuid()
base_log_dir = get_base_log_dir()
specific_log_dir = get_specific_log_dir()
storage_conf = get_storage_conf()
# Override debug considering the parameter defined in pycompss_interactive_sc script
# and exported by launch_compss
log_level = get_log_level()
if log_level == 'off':
debug = False
else:
debug = True
# Override tracing considering the parameter defined in pycompss_interactive_sc script
# and exported by launch_compss
if get_tracing():
trace = 1
else:
trace = 0
# Build a dictionary with all variables needed for initializing the runtime.
config = dict()
config['compss_home'] = compss_home
config['debug'] = debug
if project_xml is None:
project_xml = 'Runtime/configuration/xml/projects/default_project.xml'
config['project_xml'] = compss_home + os.path.sep + project_xml
else:
config['project_xml'] = project_xml
if resources_xml is None:
resources_xml = 'Runtime/configuration/xml/resources/default_resources.xml'
config['resources_xml'] = compss_home + os.path.sep + resources_xml
else:
config['resources_xml'] = resources_xml
config['summary'] = summary
config['task_execution'] = task_execution
config['storage_conf'] = storage_conf
config['task_count'] = task_count
if app_name is None:
config['app_name'] = file_name
else:
config['app_name'] = app_name
config['uuid'] = uuid
config['base_log_dir'] = base_log_dir
config['specific_log_dir'] = specific_log_dir
config['graph'] = graph
config['monitor'] = monitor
config['trace'] = trace
config['extrae_cfg'] = extrae_cfg
config['comm'] = comm
config['conn'] = conn
config['master_name'] = master_name
config['master_port'] = master_port
config['scheduler'] = scheduler
config['cp'] = cp
config['classpath'] = classpath
config['ld_library_path'] = ld_library_path
config['jvm_workers'] = jvm_workers
config['pythonpath'] = pythonpath
config['cpu_affinity'] = cpu_affinity
config['gpu_affinity'] = gpu_affinity
config['fpga_affinity'] = fpga_affinity
config['fpga_reprogram'] = fpga_reprogram
config['profile_input'] = profile_input
config['profile_output'] = profile_output
config['scheduler_config'] = scheduler_config
if external_adaptation:
config['external_adaptation'] = 'true'
else:
config['external_adaptation'] = 'false'
major_version = str(sys.version_info[0])
config['python_interpreter'] = 'python' + major_version
config['python_version'] = major_version
if 'VIRTUAL_ENV' in os.environ:
# Running within a virtual environment
python_virtual_environment = os.environ['VIRTUAL_ENV']
else:
python_virtual_environment = 'null'
config['python_virtual_environment'] = python_virtual_environment
config[
'python_propagate_virtual_environment'] = python_propagate_virtual_environment
initialize_compss(config)
# Runtime start
compss_start()
# Configure logging
app_path = app
log_path = get_log_path()
# Logging setup
if debug or log_level == "debug":
json_path = '/Bindings/python/' + major_version + '/log/logging.json.debug'
init_logging(compss_home + json_path, log_path)
elif log_level == "info":
json_path = '/Bindings/python/' + major_version + '/log/logging.json.off'
init_logging(compss_home + json_path, log_path)
elif log_level == "off":
json_path = '/Bindings/python/' + major_version + '/log/logging.json.off'
init_logging(compss_home + json_path, log_path)
else:
# Default
json_path = '/Bindings/python/' + str(
major_version) + '/log/logging.json'
init_logging(compss_home + json_path, log_path)
logger = logging.getLogger("pycompss.runtime.launch")
logger.debug('--- START ---')
logger.debug('PyCOMPSs Log path: %s' % log_path)
saved_argv = sys.argv
sys.argv = args
# Execution:
if func is None or func == '__main__':
execfile(app)
result = None
else:
import imp
imported_module = imp.load_source(file_name, app)
method_to_call = getattr(imported_module, func)
result = method_to_call(*args, **kwargs)
# Recover the system arguments
sys.argv = saved_argv
logger.debug('--- END ---')
compss_stop()
return result
def start(
log_level='off',
o_c=False,
debug=False,
graph=False,
trace=False,
monitor=None,
project_xml=None,
resources_xml=None,
summary=False,
task_execution='compss',
storage_impl=None,
storage_conf=None,
task_count=50,
app_name='Interactive',
uuid=None,
base_log_dir=None,
specific_log_dir=None,
extrae_cfg=None,
comm='NIO',
conn='es.bsc.compss.connectors.DefaultSSHConnector',
master_name='',
master_port='',
scheduler='es.bsc.compss.scheduler.loadBalancingScheduler.LoadBalancingScheduler',
jvm_workers='-Xms1024m,-Xmx1024m,-Xmn400m',
cpu_affinity='automatic',
gpu_affinity='automatic',
fpga_affinity='automatic',
fpga_reprogram='',
profile_input='',
profile_output='',
scheduler_config='',
external_adaptation=False,
propagate_virtual_environment=False,
verbose=False):
"""
Start the runtime in interactive mode.
:param log_level: Logging level [ 'on' | 'off'] (default: 'off')
:param o_c: Objects to string conversion [ True | False ] (default: False)
:param debug: Debug mode [ True | False ] (default: False)
:param graph: Generate graph [ True | False ] (default: False)
:param trace: Generate trace [ True | False ] (default: False)
:param monitor: Monitor refresh rate (default: None)
:param project_xml: Project xml file path
:param resources_xml: Resources xml file path
:param summary: Execution summary [ True | False ] (default: False)
:param task_execution: Task execution (default: 'compss')
:param storage_impl: Storage implementation path
:param storage_conf: Storage configuration file path
:param task_count: Task count (default: 50)
:param app_name: Application name (default: Interactive_date)
:param uuid: UUId
:param base_log_dir: Base logging directory
:param specific_log_dir: Specific logging directory
:param extrae_cfg: Extrae configuration file path
:param comm: Communication library (default: NIO)
:param conn: Connector (default: DefaultSSHConnector)
:param master_name: Master Name (default: '')
:param master_port: Master port (default: '')
:param scheduler: Scheduler (default: LoadBalancingScheduler)
:param jvm_workers: Java VM parameters (default: '-Xms1024m,-Xmx1024m,-Xmn400m')
:param cpu_affinity: CPU Core affinity (default: 'automatic')
:param gpu_affinity: GPU Core affinity (default: 'automatic')
:param fpga_affinity: FPA Core affinity (default: 'automatic')
:param fpga_reprogram: FPGA repogram command (default: '')
:param profile_input: Input profile (default: '')
:param profile_output: Output profile (default: '')
:param scheduler_config: Scheduler configuration (default: '')
:param external_adaptation: External adaptation [ True | False ] (default: False)
:param propagate_virtual_environment: Propagate virtual environment [ True | False ] (default: False)
:param verbose: Verbose mode [ True | False ] (default: False)
:return: None
"""
# Let the Python binding know we are at master
set_pycompss_context('MASTER')
# Prepare the environment
launch_path = os.path.dirname(os.path.realpath(__file__))
# compss_home = launch_path without the last 4 folders:
# Bindings/python/version/pycompss
compss_home = os.path.sep.join(launch_path.split(os.path.sep)[:-4])
os.environ['COMPSS_HOME'] = compss_home
# Get environment variables
cp = os.getcwd() + '/'
pythonpath = os.environ['PYTHONPATH']
classpath = os.environ['CLASSPATH']
ld_library_path = os.environ['LD_LIBRARY_PATH']
# Set storage classpath
if storage_impl:
if storage_impl == 'redis':
cp = cp + ':' + compss_home + '/Tools/storage/redis/compss-redisPSCO.jar'
else:
cp = cp + ':' + storage_impl
# Set extrae dependencies
if not "EXTRAE_HOME" in os.environ:
# It can be defined by the user or by launch_compss when running in Supercomputer
extrae_home = compss_home + '/Dependencies/extrae'
os.environ['EXTRAE_HOME'] = extrae_home
else:
extrae_home = os.environ['EXTRAE_HOME']
extrae_lib = extrae_home + '/lib'
# Include extrae into ld_library_path
ld_library_path = extrae_lib + ':' + ld_library_path
os.environ['LD_LIBRARY_PATH'] = ld_library_path
if monitor is not None:
# Enable the graph if the monitoring is enabled
graph = True
# Export global variables
global graphing
graphing = graph
__export_globals__()
print("******************************************************")
print("*************** PyCOMPSs Interactive *****************")
print("******************************************************")
print("* .-~~-.--. ____ ____ *")
print("* : ) |___ \ |___ \ *")
print("* .~ ~ -.\ /.- ~~ . __) | __) | *")
print("* > `. .' < / __/ _ ___) | *")
print("* ( .- -. ) |_____| |_| |____/ *")
print("* `- -.-~ `- -' ~-.- -' *")
print("* ( : ) _ _ .-: *")
print("* ~--. : .--~ .-~ .-~ } *")
print("* ~-.-^-.-~ \_ .~ .-~ .~ *")
print("* \ \ ' \ '_ _ -~ *")
print("* \`.\`. // *")
print("* . - ~ ~-.__\`.\`-.// *")
print("* .-~ . - ~ }~ ~ ~-.~-. *")
print("* .' .-~ .-~ :/~-.~-./: *")
print("* /_~_ _ . - ~ ~-.~-._ *")
print("* ~-.< *")
print("******************************************************")
##############################################################
# INITIALIZATION
##############################################################
if RUNNING_IN_SUPERCOMPUTER:
# Since the deployment in supercomputers is done through the use of enqueue_compss
# and consequently launch_compss - the project and resources xmls are already created
project_xml, resources_xml = get_xmls()
# It also exported some environment variables that we need here
master_name = get_master_node()
master_port = get_master_port()
uuid = get_uuid()
base_log_dir = get_base_log_dir()
specific_log_dir = get_specific_log_dir()
storage_conf = get_storage_conf()
# Override debug considering the parameter defined in pycompss_interactive_sc script
# and exported by launch_compss
log_level = get_log_level()
if log_level == 'off':
debug = False
else:
debug = True
# Override tracing considering the parameter defined in pycompss_interactive_sc script
# and exported by launch_compss
trace = get_tracing()
if verbose:
print("- Overridden project xml with: " + project_xml)
print("- Overridden resources xml with: " + resources_xml)
print("- Overridden master name with: " + master_name)
print("- Overridden master port with: " + master_port)
print("- Overridden uuid with: " + uuid)
print("- Overridden base log dir with: " + base_log_dir)
print("- Overridden specific log dir with: " + specific_log_dir)
print("- Overridden storage conf with: " + storage_conf)
print("- Overridden log level with: " + str(log_level))
print("- Overridden debug with: " + str(debug))
print("- Overridden trace with: " + str(trace))
if debug:
# Add environment variable to get binding-commons debug information
os.environ['COMPSS_BINDINGS_DEBUG'] = '1'
if trace is False:
trace = 0
elif trace == 'basic' or trace is True:
trace = 1
os.environ['LD_PRELOAD'] = extrae_lib + '/libpttrace.so'
elif trace == 'advanced':
trace = 2
os.environ['LD_PRELOAD'] = extrae_lib + '/libpttrace.so'
else:
print(
"ERROR: Wrong tracing parameter ( [ True | basic ] | advanced | False)"
)
return -1
# Build a dictionary with all variables needed for initializing the runtime
config = dict()
config['compss_home'] = compss_home
config['debug'] = debug
if project_xml is None:
project_path = 'Runtime/configuration/xml/projects/default_project.xml'
config['project_xml'] = compss_home + os.path.sep + project_path
else:
config['project_xml'] = project_xml
if resources_xml is None:
resources_path = 'Runtime/configuration/xml/resources/default_resources.xml'
config['resources_xml'] = compss_home + os.path.sep + resources_path
else:
config['resources_xml'] = resources_xml
config['summary'] = summary
config['task_execution'] = task_execution
config['storage_conf'] = storage_conf
config['task_count'] = task_count
if app_name is None:
config['app_name'] = 'Interactive'
else:
config['app_name'] = app_name
config['uuid'] = uuid
config['base_log_dir'] = base_log_dir
config['specific_log_dir'] = specific_log_dir
config['graph'] = graph
config['monitor'] = monitor
config['trace'] = trace
config['extrae_cfg'] = extrae_cfg
config['comm'] = comm
config['conn'] = conn
config['master_name'] = master_name
config['master_port'] = master_port
config['scheduler'] = scheduler
config['cp'] = cp
config['pythonpath'] = pythonpath
config['classpath'] = classpath
config['ld_library_path'] = ld_library_path
config['jvm_workers'] = jvm_workers
config['cpu_affinity'] = cpu_affinity
config['gpu_affinity'] = gpu_affinity
config['fpga_affinity'] = fpga_affinity
config['fpga_reprogram'] = fpga_reprogram
config['profile_input'] = profile_input
config['profile_output'] = profile_output
config['scheduler_config'] = scheduler_config
if external_adaptation:
config['external_adaptation'] = 'true'
else:
config['external_adaptation'] = 'false'
major_version = sys.version_info[0]
python_interpreter = 'python' + str(major_version)
config['python_interpreter'] = python_interpreter
config['python_version'] = str(major_version)
if 'VIRTUAL_ENV' in os.environ:
# Running within a virtual environment
python_virtual_environment = os.environ['VIRTUAL_ENV']
else:
python_virtual_environment = 'null'
config['python_virtual_environment'] = python_virtual_environment
config[
'python_propagate_virtual_environment'] = propagate_virtual_environment
initialize_compss(config)
##############################################################
# RUNTIME START
##############################################################
print("* - Starting COMPSs runtime... *")
sys.stdout.flush() # Force flush
compss_start()
if o_c is True:
# set cross-module variable
binding.object_conversion = True
else:
# set cross-module variable
binding.object_conversion = False
# Remove launch.py, log_level and object_conversion from sys.argv,
# It will be inherited by the app through execfile
# sys.argv = sys.argv[3:]
# Get application execution path
# app_path = sys.argv[0] # not needed in interactive mode
global log_path
log_path = get_log_path()
binding.temp_dir = mkdtemp(prefix='pycompss', dir=log_path + '/tmpFiles/')
print("* - Log path : " + log_path)
# Logging setup - messages before this step are ignored (need log_path to configure the logger).
if debug or log_level == "debug":
json_path = '/Bindings/python/' + str(
major_version) + '/log/logging.json.debug'
init_logging(os.getenv('COMPSS_HOME') + json_path, log_path)
elif log_level == "info":
json_path = '/Bindings/python/' + str(
major_version) + '/log/logging.json.off'
init_logging(os.getenv('COMPSS_HOME') + json_path, log_path)
elif log_level == "off":
json_path = '/Bindings/python/' + str(
major_version) + '/log/logging.json.off'
init_logging(os.getenv('COMPSS_HOME') + json_path, log_path)
else:
# Default
json_path = '/Bindings/python/' + str(
major_version) + '/log/logging.json'
init_logging(os.getenv('COMPSS_HOME') + json_path, log_path)
logger = logging.getLogger("pycompss.runtime.launch")
__print_setup__(verbose, log_level, o_c, debug, graph, trace, monitor,
project_xml, resources_xml, summary, task_execution,
storage_conf, pythonpath, classpath, ld_library_path,
task_count, app_name, uuid, base_log_dir, specific_log_dir,
extrae_cfg, comm, conn, master_name, master_port,
scheduler, jvm_workers, cpu_affinity, gpu_affinity,
fpga_affinity, fpga_reprogram, profile_input,
profile_output, scheduler_config, external_adaptation,
python_interpreter, major_version,
python_virtual_environment, propagate_virtual_environment)
logger.debug("--- START ---")
logger.debug("PyCOMPSs Log path: %s" % log_path)
if storage_conf is not None:
logger.debug("Storage configuration file: %s" % storage_conf)
from storage.api import init as init_storage
init_storage(config_file_path=storage_conf)
global persistent_storage
persistent_storage = True
# MAIN EXECUTION
# let the user write an interactive application
print("* - PyCOMPSs Runtime started... Have fun! *")
print("******************************************************")
# Instance method
file_name = values.pop()
f = open(file_name, 'rb')
obj = load(f)
logger.debug("Processing callee, a hidden object of %s in file %s" % (file_name, type(obj)))
values.insert(0, obj)
types.pop()
types.insert(0, Type.OBJECT)
f.close()
getattr(klass, method_name)(*values, compss_types = types)
f = open(file_name, 'wb')
dump(obj, f, HIGHEST_PROTOCOL)
f.close()
else:
# Class method - class is not included in values (e.g. values = [7])
types.insert(0, None) # class must be first type
getattr(klass, method_name)(*values, compss_types = types)
except Exception:
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
logger.exception("WORKER EXCEPTION")
logger.exception(''.join(line for line in lines))
exit(1)
if __name__ == "__main__":
init_logging(os.getenv('PYCOMPSS_HOME') + '/log/logging.json')
compss_worker()
types.ListType : Type.OBJECT, # The type of lists (e.g. [0, 1, 2, 3])
types.DictType : Type.OBJECT # The type of dictionaries (e.g. {'Bacon': 1, 'Ham': 0})
}
temp_dir = mkdtemp(prefix = 'pycompss', dir = os.getcwd())
temp_obj_prefix = "/compss-serialized-obj_"
objid_to_filename = {}
task_objects = {}
# Objects that have been accessed by the main program
objs_written_by_mp = {} # obj_id -> compss_file_name
init_logging(os.getenv('IT_HOME') + '/bindings/python/log/logging.json')
logger = logging.getLogger(__name__)
class Function_Type:
FUNCTION = 1
INSTANCE_METHOD = 2
CLASS_METHOD = 3
class Future(object):
pass
def start_runtime():
compss.start_runtime()
logger.info("COMPSs started")
