def run_cluster_vessmorphovis(arguments):
"""Run the VessMorphoVis framework on the BBP visualization cluster using SLURM.
:param arguments:
Command line arguments.
"""
# Use the morphology file (.H5 or .VMV)
if arguments.input == 'file':
# Get the arguments string list
arguments_string = arguments_parser.get_arguments_string(
arguments=arguments)
# Run the job on the cluster
slurm.run_morphology_files_jobs_on_cluster(
arguments=arguments, morphology_files=arguments.morphology_file)
# Operate on a directory
elif arguments.input == 'directory':
# Get all the morphology files in this directory
morphology_files = file_ops.get_files_in_directory(
arguments.morphology_directory, '.h5')
# Run the jobs on the cluster
slurm.run_morphology_files_jobs_on_cluster(
arguments=arguments, morphology_files=morphology_files)
else:
print('ERROR: Input data source, use [file, gid, target or directory]')
exit(0)
def run_local_neuromorphovis(arguments):
"""Run the framework on a local node, basically your machine.
:param arguments:
Command line arguments.
"""
# Retrieve the path to the interface
current_path = os.path.dirname(os.path.realpath(__file__))
cli_interface = '%s/neuromorphovis/interface/cli/cli_interface.py' % current_path
# Target and GID options are only available on the BBP visualization clusters
if arguments.input == 'target' or arguments.input == 'gid':
print(
'ERROR, Target and GID options are only available on the BBP visualization clusters'
)
exit(0)
# Load morphology files (.H5 or .SWC)
elif arguments.input == 'file':
# Get the arguments string list
arguments_string = arguments_parser.get_arguments_string(
arguments=arguments)
# NOTE: Using a morphology file, either in .H5 or .SWC formats is straightforward and
# therefore the arguments will not change at all. In this case it is safe to pass the
# arguments as they were received without any change.
# Construct the shell command to run the workflow
# -b : Blender background mode
# --verbose : Turn off all the verbose messages
# -- : Separate the framework arguments from those given to Blender
shell_command = '%s -b --verbose 0 --python %s -- %s ' % (
arguments.blender, cli_interface, arguments_string)
# Run NeuroMorphoVis from Blender in the background mode
print('RUNNING: ' + shell_command)
subprocess.call(shell_command, shell=True)
# Load a directory morphology files (.H5 or .SWC)
elif arguments.input == 'directory':
# Get all the morphology files in this directory
# TODO: Verify the installation of H5Py before running the workflow
# TODO: Add the support to load .SWC files
morphology_files = \
file_ops.get_files_in_directory(arguments.morphology_directory, '.h5')
# If the directory is empty, give an error message
if len(morphology_files) == 0:
print(
'ERROR: The directory [%s] does NOT contain any morphology files'
% arguments.morphology_directory)
# Construct the commands for every individual morphology file
for morphology_file in morphology_files:
# Get the argument string for an individual file
arguments_string = arguments_parser.get_arguments_string_for_individual_file(
arguments=arguments, morphology_file=morphology_file)
# Construct the shell command to run the workflow
# -b : Blender background mode
# --verbose : Turn off all the verbose messages
# -- : Separate the framework arguments from those given to Blender
shell_command = '%s -b --verbose 0 --python %s -- %s ' % (
arguments.blender, cli_interface, arguments_string)
# Run NeuroMorphoVis from Blender in the background mode
print('RUNNING: ' + shell_command)
subprocess.call(shell_command, shell=True)
else:
print(
'ERROR: Input data source, use \'file, gid, target or directory\'')
exit(0)
def run_cluster_neuromorphovis(arguments):
"""Run the NeuroMorphoVis framework on the BBP visualization cluster using SLURM.
:param arguments:
Command line arguments.
"""
# Use a specific circuit target
if arguments.input == 'target':
# Log
print('Loading a target [%s] in circuit [%s]' %
(arguments.target, arguments.blue_config))
# Ensure a valid blue config and a target
if arguments.blue_config is None or arguments.target is None:
print('ERROR: Empty circuit configuration file or target')
exit(0)
# Import brain
try:
import brain
except ImportError:
print(
'ERROR: Cannot import [brain], please load brain or install it'
)
exit(0)
# Open a circuit with a given blue config
bbp_circuit = brain.Circuit(arguments.blue_config)
# Create a GID-set and load the morphologies from these GIDs
gids = bbp_circuit.gids(arguments.target)
# Run the jobs on the cluster
slurm.run_gid_jobs_on_cluster(arguments=arguments, gids=gids)
# Use a single GID
elif arguments.input == 'gid':
print('Loading a gid [%s] in circuit [%s]' %
(str(arguments.gid), arguments.blue_config))
# Ensure a valid blue config and a GID
if arguments.blue_config is None or arguments.gid is None:
print('ERROR: Empty circuit configuration file or GID')
exit(0)
# Run the jobs on the cluster
slurm.run_gid_jobs_on_cluster(arguments=arguments,
gids=[str(arguments.gid)])
# Use the morphology file (.H5 or .SWC)
elif arguments.input == 'file':
# Get the arguments string list
arguments_string = arguments_parser.get_arguments_string(
arguments=arguments)
# Run the job on the cluster
slurm.run_morphology_files_jobs_on_cluster(
arguments=arguments, morphology_files=arguments.morphology_file)
# Operate on a directory
elif arguments.input == 'directory':
# Get all the morphology files in this directory
morphology_files = file_ops.get_files_in_directory(
arguments.morphology_directory, '.h5')
# Run the jobs on the cluster
slurm.run_morphology_files_jobs_on_cluster(
arguments=arguments, morphology_files=morphology_files)
else:
print('ERROR: Input data source, use [file, gid, target or directory]')
exit(0)
def run_local_neuromorphovis(arguments):
"""Run the framework on a local node, basically your machine.
:param arguments:
Command line arguments.
"""
# Target and GID options are only available on the BBP visualization clusters
if arguments.input == 'target' or arguments.input == 'gid':
print(
'ERROR, Target and GID options are only available on the BBP visualization clusters'
)
exit(0)
# Load morphology files (.H5 or .SWC)
elif arguments.input == 'file':
# Get the arguments string list
arguments_string = arguments_parser.get_arguments_string(
arguments=arguments)
# NOTE: Using a morphology file, either in .H5 or .SWC formats is straightforward and
# therefore the arguments will not change at all. In this case it is safe to pass the
# arguments as they were received without any change.
# Construct the shell command to run the workflow
shell_commands = create_shell_commands_for_local_execution(
arguments, arguments_string)
# Run NeuroMorphoVis from Blender in the background mode
for shell_command in shell_commands:
print('RUNNING: ' + shell_command)
subprocess.call(shell_command, shell=True)
# Load a directory morphology files (.H5 or .SWC)
elif arguments.input == 'directory':
# Get all the morphology files in this directory
# TODO: Verify the installation of H5Py before running the workflow
# TODO: Add the support to load .SWC files from a directory at the same time
morphology_files = file_ops.get_files_in_directory(
arguments.morphology_directory, '.h5')
# If the directory is empty, give an error message
if len(morphology_files) == 0:
print(
'ERROR: The directory [%s] does NOT contain any morphology files'
% arguments.morphology_directory)
# A list of all the commands to be executed
shell_commands = list()
# Construct the commands for every individual morphology file
for morphology_file in morphology_files:
# Get the argument string for an individual file
arguments_string = arguments_parser.get_arguments_string_for_individual_file(
arguments=arguments, morphology_file=morphology_file)
# Construct the shell command to run the workflow
shell_commands.extend(
create_shell_commands_for_local_execution(
arguments, arguments_string))
# Run NeuroMorphoVis from Blender in the background mode
for shell_command in shell_commands:
# print('RUNNING: ' + shell_command)
subprocess.call(shell_command, shell=True)
else:
print(
'ERROR: Input data source, use \'file, gid, target or directory\'')
exit(0)
def run_local_neuromorphovis(arguments):
"""Run the framework on a local node, basically your machine.
:param arguments:
Command line arguments.
"""
# Use a specific circuit target
if arguments.input == 'target':
# Log
print('Loading a target [%s] in circuit [%s]' % (arguments.target, arguments.blue_config))
# Ensure a valid blue config and a target
if arguments.blue_config is None or arguments.target is None:
print('ERROR: Empty circuit configuration file or target')
exit(0)
# Import BluePy
try:
import bluepy.v2
except ImportError:
print('ERROR: Cannot import [BluePy], please install it')
exit(0)
from bluepy.v2 import Circuit
# Loading a circuit
circuit = Circuit(arguments.blue_config)
# Loading the GIDs of the sample target within the circuit
gids = circuit.cells.ids(arguments.target)
shell_commands = list()
for gid in gids:
# Get the argument string for an individual file
arguments_string = arguments_parser.get_arguments_string_for_individual_gid(
arguments=arguments, gid=gid)
# Construct the shell command to run the workflow
shell_commands.extend(
create_shell_commands_for_local_execution(arguments, arguments_string))
# Run NeuroMorphoVis from Blender in the background mode
for shell_command in shell_commands:
subprocess.call(shell_command, shell=True)
# Use a single GID
elif arguments.input == 'gid':
print('Loading a gid [%s] in circuit [%s]' % (str(arguments.gid), arguments.blue_config))
# Ensure a valid blue config and a GID
if arguments.blue_config is None or arguments.gid is None:
print('ERROR: Empty circuit configuration file or GID')
exit(0)
# Get the argument string for an individual file
arguments_string = arguments_parser.get_arguments_string_for_individual_gid(
arguments=arguments, gid=arguments.gid)
# Construct the shell command to run the workflow
shell_commands = create_shell_commands_for_local_execution(arguments, arguments_string)
# Run NeuroMorphoVis from Blender in the background mode
for shell_command in shell_commands:
subprocess.call(shell_command, shell=True)
# Run the jobs on the cluster
slurm.run_gid_jobs_on_cluster(arguments=arguments, gids=[str(arguments.gid)])
# Load morphology files (.H5 or .SWC)
elif arguments.input == 'file':
# Get the arguments string list
arguments_string = arguments_parser.get_arguments_string(arguments=arguments)
# NOTE: Using a morphology file, either in .H5 or .SWC formats is straightforward and
# therefore the arguments will not change at all. In this case it is safe to pass the
# arguments as they were received without any change.
# Construct the shell command to run the workflow
shell_commands = create_shell_commands_for_local_execution(arguments, arguments_string)
# Run NeuroMorphoVis from Blender in the background mode
for shell_command in shell_commands:
subprocess.call(shell_command, shell=True)
# Load a directory morphology files (.H5 or .SWC)
elif arguments.input == 'directory':
# Get all the morphology files in this directory
morphology_files = file_ops.get_files_in_directory(arguments.morphology_directory, '.h5')
morphology_files.extend(file_ops.get_files_in_directory(
arguments.morphology_directory, '.swc'))
# If the directory is empty, give an error message
if len(morphology_files) == 0:
print('ERROR: The directory [%s] does NOT contain any morphology files' %
arguments.morphology_directory)
# A list of all the commands to be executed
shell_commands = list()
# Construct the commands for every individual morphology file
for morphology_file in morphology_files:
# Get the argument string for an individual file
arguments_string = arguments_parser.get_arguments_string_for_individual_file(
arguments=arguments, morphology_file=morphology_file)
# Construct the shell command to run the workflow
shell_commands.extend(
create_shell_commands_for_local_execution(arguments, arguments_string))
# Run NeuroMorphoVis from Blender in the background mode
for shell_command in shell_commands:
print('RUNNING: **********************************************************************')
print(shell_command)
print('*******************************************************************************')
subprocess.call(shell_command, shell=True)
else:
print('ERROR: Input data source, use \'file, gid, target or directory\'')
exit(0)
def submit_batch_jobs(user_name, slurm_jobs_directory):
"""Submits all the batch jobs found in the jobs directory.
This function takes into account the maximum limit imposed by the cluster (500 jobs per user).
:param user_name:
The user name of the current user.
:param slurm_jobs_directory:
The directory where the batch jobs are created. .
"""
# Get all the scripts in the slurm jobs directory to submit them
scripts = file_ops.get_files_in_directory(slurm_jobs_directory,
file_extension='.sh')
# Use an index to keep track on the number of jobs submitted to the cluster.
script_index = 0
# Submit the jobs taking into account the maximum number of jobs dedicated per user
while True:
# Just sleep for a second
time.sleep(1)
# Get the number of jobs active for that user
number_active_jobs = get_current_number_jobs_for_user(
user_name=user_name)
# If the number of jobs is greater than 500, then wait a second and try again
if number_active_jobs >= 500:
print('Waiting for resources ...')
continue
# Otherwise, you can submit some jobs
else:
# Get the number of jobs that are available to submit
number_available_jobs = 500 - number_active_jobs
# Submit as many jobs as you can
for i in range(number_available_jobs):
# Make sure that we still have some jobs to submit, otherwise break
if script_index >= len(scripts):
return
# Get the script full path
script_full_path = '%s/%s' % (slurm_jobs_directory,
scripts[script_index])
# 'chmod' the script to be able to execute it
shell_command = 'chmod +x %s' % script_full_path
# Execute the command
subprocess.call(shell_command, shell=True)
# Format the shell command
shell_command = 'sbatch %s' % script_full_path
# Execute the command
print('Submitting [%s]' % shell_command)
subprocess.call(shell_command, shell=True)
# Increment the script index
script_index += 1
