def run_operation(default_backend, operation):
""" Runs the given operation on the backend
Runs the given operation *operation* either on the backend specified in the
operation' spec file or (if none is specified) on the backend
passed as *default_backend*.
"""
# Check if default backend can be used or if we have to run on a separate
# backend
if "backend" in operation.operation_spec:
backend = create_backend(operation.operation_spec["backend"])
print(" --> For current operation using backend: \n\t\t %s." %
str(backend))
else:
backend = default_backend
# In case a operation_chain is executed the queue needs to be reset, since
# the the first terminated operation cleans and closes the queue.
if backend.__str__() == "MulticoreBackend":
backend.reset_queue()
backend.stage_in(operation)
backend.execute()
backend.retrieve()
backend.consolidate()
output_directory = operation.get_output_directory()
backend.cleanup()
return output_directory
def run_operation(default_backend, operation, timeout=None):
""" Runs the given operation on the backend
Runs the given operation *operation* either on the backend specified in the
operation' spec file or (if none is specified) on the backend
passed as *default_backend*.
"""
# Check if default backend can be used or if we have to run on a separate
# backend
if "backend" in operation.operation_spec:
backend = create_backend(operation.operation_spec["backend"])
print(" --> For current operation using backend: \n\t\t %s."%str(backend))
else:
backend = default_backend
# In case a operation_chain is executed the queue needs to be reset, since
# the the first terminated operation cleans and closes the queue.
if backend.__str__() == "MulticoreBackend":
backend.reset_queue()
backend.stage_in(operation)
backend.execute()
if backend.retrieve(timeout=timeout):
backend.consolidate()
output_directory = operation.get_output_directory()
else:
output_directory = None
backend.cleanup()
return output_directory
def run_operation(default_backend, operation, ex_timeout=1e6, re_timeout=1e6):
""" Runs the given operation on the backend
Runs the given operation *operation* either on the backend specified in the
operation' spec file or (if none is specified) on the backend
passed as *default_backend*.
Different timeouts are required, because for the execute function get is
called which does not accept to high timeouts without proper error handling
on a Mac OS X whereas Linux systems are fine with larger timeouts.
"""
# Check if default backend can be used or if we have to run on a separate
# backend
if "backend" in operation.operation_spec:
backend = create_backend(operation.operation_spec["backend"])
LOGGER.info(" --> For current operation using backend: \n\t\t %s."%str(backend))
else:
backend = default_backend
# In case a operation_chain is executed the queue needs to be reset, since
# the the first terminated operation cleans and closes the queue.
if backend.__str__() == "MulticoreBackend":
backend.reset_queue()
backend.stage_in(operation)
try:
backend.execute(timeout=ex_timeout)
backend.retrieve(timeout=re_timeout)
backend.consolidate()
return operation.get_output_directory()
finally:
backend.cleanup()
def run_operation(default_backend, operation, ex_timeout=1e6, re_timeout=1e6):
""" Runs the given operation on the backend
Runs the given operation *operation* either on the backend specified in the
operation' spec file or (if none is specified) on the backend
passed as *default_backend*.
Different timeouts are required, because for the execute function get is
called which does not accept to high timeouts without proper error handling
on a Mac OS X whereas Linux systems are fine with larger timeouts.
"""
# Check if default backend can be used or if we have to run on a separate
# backend
if "backend" in operation.operation_spec:
backend = create_backend(operation.operation_spec["backend"])
LOGGER.info(" --> For current operation using backend: \n\t\t %s." %
str(backend))
else:
backend = default_backend
# In case a operation_chain is executed the queue needs to be reset, since
# the the first terminated operation cleans and closes the queue.
if backend.__str__() == "MulticoreBackend":
backend.reset_queue()
backend.stage_in(operation)
try:
backend.execute(timeout=ex_timeout)
backend.retrieve(timeout=re_timeout)
backend.consolidate()
return operation.get_output_directory()
finally:
backend.cleanup()
def _start(self):
# Set number of processes in operation
self.progressBar.setMinimum(0)
self.progressBar.setMaximum(self.operation.number_processes)
self.progressBar.setValue(0)
# Create backend
self.backend = create_backend(str(self.backendComboBox.currentText()))
# Run operation in a separate thread to keep GUI responsive
self.operationThread = Thread(target=pySPACE.run_operation,
args=(self.backend, self.operation))
self.operationThread.start()
# Update progress bar in a separate thread
self.progressBarThread = Thread(target=self._monitor_progress, args=())
self.progressBarThread.start()
def main():
#### Find pySPACE package and import it ####
# Determine path of current file
path = os.path.realpath(__file__)
# Move up to parent directory that contains the pySPACE tree
suffix = []
for i in range(3):
path, tail = os.path.split(path)
suffix.append(tail)
parent_dir = path
# Check proper directory structure
if suffix != ['launch.py', 'run', 'pySPACE']:
raise RuntimeError, "Encountered incorrect directory structure. "\
"launch.py needs to reside in $PARENT_DIR/pySPACE/run"
# Workaround for eegserver crashing after 255 open ports
# - Now it crashes after 4096 open ports ;-)
import resource
(fd1, fd2) = resource.getrlimit(resource.RLIMIT_NOFILE)
fd1 = 4096 if fd2 == resource.RLIM_INFINITY else fd2 - 1
resource.setrlimit(resource.RLIMIT_NOFILE, (fd1, fd2))
# ------------------------------------------------------
#########################################
### Parsing of command line arguments
usage = "Usage: %prog [BACKEND_SPECIFICATION] [--config <conf.yaml>] "\
"[--operation <operation.yaml> | --operation_chain <operation_chain.yaml>] "\
"[--profile]"\
" where BACKEND_SPECIFICATION can be --serial, --mcore, --loadl or --mpi"
parser = LaunchParser(usage=usage, epilog=epilog)
# Configuration
parser.add_option(
"-c",
"--configuration",
help=
"Choose the configuration file, which is looked up in PYSPACE_CONF_DIR",
action="store")
# Backends
parser.add_option(
"-s",
"--serial",
action="store_true",
default=False,
help="Enables execution on the SerialBackend (one local process)")
parser.add_option(
"-m",
"--mcore",
action="store_true",
default=False,
help=
"Enables execution on the MulticoreBackend (one process per CPU core)")
parser.add_option(
"-l",
"--local",
action="store_true",
default=False,
help=
"Enables execution on the MulticoreBackend (one process per CPU core)")
parser.add_option("-i",
"--mpi",
action="store_true",
default=False,
help="Enables execution via MPI")
parser.add_option("-L",
"--loadl",
action="store_true",
default=False,
help="Enables execution via LoadLeveler.")
# Operation / operation chain
parser.add_option("-o",
"--operation",
help="Chooses the operation that will be executed. The "
"operation specification file is looked up in "
"$SPEC_DIR/operations",
action="store")
parser.add_option(
"-O",
"-C",
"--operation_chain",
help="Chooses the operation chain that will be executed. "
"The operation chain specification file is looked up "
"in $SPEC_DIR/operation_chains",
action="store")
# Profiling
parser.add_option(
"-p",
"--profile",
help="Profiles execution.",
action="store_true",
default=False,
)
(options, args) = parser.parse_args()
# Load configuration file
pySPACE.load_configuration(options.configuration)
if hasattr(pySPACE.configuration, "eeg_acquisition_dir"):
eeg_parent_dir =\
os.sep.join(pySPACE.configuration.eeg_acquisition_dir.split(os.sep)[:-1])
if not hasattr(pySPACE.configuration, "eeg_acquisition_dir"):
pySPACE.configuration.eeg_module_path = eeg_parent_dir
else:
eeg_parent_dir, tail = os.path.split(parent_dir)
eeg_parent_dir = os.path.join(eeg_parent_dir, "eeg_modules")
pySPACE.configuration.eeg_module_path = eeg_parent_dir
sys.path.append(eeg_parent_dir)
# Create backend
if options.serial:
default_backend = create_backend("serial")
elif options.mcore or options.local:
default_backend = create_backend("mcore")
elif options.mpi:
default_backend = create_backend("mpi")
elif options.loadl:
default_backend = create_backend("loadl")
else: # Falling back to serial backend
default_backend = create_backend("serial")
print(" --> Using backend: \n\t\t %s." % str(default_backend))
if not options.operation is None:
# Create operation for the given name
operation = create_operation_from_file(options.operation)
# Store current source code for later inspection
create_source_archive(archive_path=operation.get_output_directory())
if not options.profile:
# Execute the current operation
run_operation(default_backend, operation)
else:
# Execute and profile operation
cProfile.runctx('pySPACE.run_operation(default_backend, operation)',
globals(), locals(),
filename = operation.get_output_directory()\
+ os.sep + "profile.pstat")
elif not options.operation_chain is None:
# Create operation chain for the given name
operation_chain = create_operation_chain(options.operation_chain)
# Store current source code for later inspection
create_source_archive(
archive_path=operation_chain.get_output_directory())
if not options.profile:
# Execute the current operation_chain
run_operation_chain(default_backend, operation_chain)
else:
# Execute and profile operation
cProfile.runctx('pySPACE.run_operation_chain(default_backend, operation_chain)',
globals(), locals(),
filename=operation_chain.get_output_directory()\
+ os.sep + "profile.pstat")
else:
parser.error(
"Neither operation chain nor operation specification file given!")
logging.shutdown()
# Stop logger thread in backend
default_backend._stop_logging()
del default_backend
def main():
#### Find pySPACE package and import it ####
# Determine path of current file
path = os.path.realpath(__file__)
# Move up to parent directory that contains the pySPACE tree
suffix = []
for i in range(3):
path, tail = os.path.split(path)
suffix.append(tail)
parent_dir = path
# Check proper directory structure
if suffix != ['launch.py', 'run', 'pySPACE']:
raise RuntimeError, "Encountered incorrect directory structure. "\
"launch.py needs to reside in $PARENT_DIR/pySPACE/run"
# Workaround for eegserver crashing after 255 open ports
# - Now it crashes after 4096 open ports ;-)
import resource
(fd1, fd2) = resource.getrlimit(resource.RLIMIT_NOFILE)
fd1 = 4096 if fd2 == resource.RLIM_INFINITY else fd2-1
resource.setrlimit(resource.RLIMIT_NOFILE, (fd1,fd2))
# ------------------------------------------------------
#########################################
### Parsing of command line arguments
usage = "Usage: %prog [BACKEND_SPECIFICATION] [--config <conf.yaml>] "\
"[--operation <operation.yaml> | --operation_chain <operation_chain.yaml>] "\
"[--profile]"\
" where BACKEND_SPECIFICATION can be --serial, --mcore, --loadl or --mpi"
parser = LaunchParser(usage=usage, epilog=epilog)
# Configuration
parser.add_option("-c", "--configuration",
default="config.yaml",
help="Choose the configuration file, which is looked up in PYSPACE_CONF_DIR",
action="store")
# Backends
parser.add_option("-s", "--serial", action="store_true", default=False,
help="Enables execution on the SerialBackend (one local process)")
parser.add_option("-m", "--mcore", action="store_true", default=False,
help="Enables execution on the MulticoreBackend (one process per CPU core)")
parser.add_option("-l", "--local", action="store_true", default=False,
help="Enables execution on the MulticoreBackend (one process per CPU core)")
parser.add_option("-i", "--mpi", action="store_true", default=False,
help="Enables execution via MPI")
parser.add_option("-L", "--loadl", action="store_true", default=False,
help="Enables execution via LoadLeveler.")
# Operation / operation chain
parser.add_option("-o", "--operation",
help="Chooses the operation that will be executed. The "
"operation specification file is looked up in "
"$SPEC_DIR/operations",
action="store")
parser.add_option("-O", "-C", "--operation_chain",
help="Chooses the operation chain that will be executed. "
"The operation chain specification file is looked up "
"in $SPEC_DIR/operation_chains",
action="store")
# Profiling
parser.add_option("-p", "--profile",
help="Profiles execution.",
action="store_true", default=False,)
(options, args) = parser.parse_args()
# Load configuration file
pySPACE.load_configuration(options.configuration)
if hasattr(pySPACE.configuration, "eeg_acquisition_dir"):
eeg_parent_dir =\
os.sep.join(pySPACE.configuration.eeg_acquisition_dir.split(os.sep)[:-1])
if not hasattr(pySPACE.configuration, "eeg_acquisition_dir"):
pySPACE.configuration.eeg_module_path = eeg_parent_dir
else:
eeg_parent_dir, tail = os.path.split(parent_dir)
eeg_parent_dir = os.path.join(eeg_parent_dir, "eeg_modules")
pySPACE.configuration.eeg_module_path = eeg_parent_dir
sys.path.append(eeg_parent_dir)
# Create backend
if options.serial:
default_backend = create_backend("serial")
elif options.mcore or options.local:
default_backend = create_backend("mcore")
elif options.mpi:
default_backend = create_backend("mpi")
elif options.loadl:
default_backend = create_backend("loadl")
else: # Falling back to serial backend
default_backend = create_backend("serial")
print(" --> Using backend: \n\t\t %s."%str(default_backend))
if not options.operation is None:
# Create operation for the given name
operation = create_operation_from_file(options.operation)
# Store current source code for later inspection
create_source_archive(archive_path=operation.get_output_directory())
if not options.profile:
# Execute the current operation
run_operation(default_backend, operation)
else:
# Execute and profile operation
cProfile.runctx('pySPACE.run_operation(default_backend, operation)',
globals(), locals(),
filename = operation.get_output_directory()\
+ os.sep + "profile.pstat")
elif not options.operation_chain is None:
# Create operation chain for the given name
operation_chain = create_operation_chain(options.operation_chain)
# Store current source code for later inspection
create_source_archive(archive_path=operation_chain.get_output_directory())
if not options.profile:
# Execute the current operation_chain
run_operation_chain(default_backend, operation_chain)
else:
# Execute and profile operation
cProfile.runctx('pySPACE.run_operation_chain(default_backend, operation_chain)',
globals(), locals(),
filename=operation_chain.get_output_directory()\
+ os.sep + "profile.pstat")
else:
parser.error("Neither operation chain nor operation specification file given!")
logging.shutdown()
# Stop logger thread in backend
default_backend._stop_logging()
del default_backend