def run_cluster_vessmorphovis(arguments): """Run the VessMorphoVis framework on the BBP visualization cluster using SLURM. :param arguments: Command line arguments. """ # Use the morphology file (.H5 or .VMV) if arguments.input == 'file': # Get the arguments string list arguments_string = arguments_parser.get_arguments_string( arguments=arguments) # Run the job on the cluster slurm.run_morphology_files_jobs_on_cluster( arguments=arguments, morphology_files=arguments.morphology_file) # Operate on a directory elif arguments.input == 'directory': # Get all the morphology files in this directory morphology_files = file_ops.get_files_in_directory( arguments.morphology_directory, '.h5') # Run the jobs on the cluster slurm.run_morphology_files_jobs_on_cluster( arguments=arguments, morphology_files=morphology_files) else: print('ERROR: Input data source, use [file, gid, target or directory]') exit(0)
def run_local_neuromorphovis(arguments): """Run the framework on a local node, basically your machine. :param arguments: Command line arguments. """ # Retrieve the path to the interface current_path = os.path.dirname(os.path.realpath(__file__)) cli_interface = '%s/neuromorphovis/interface/cli/cli_interface.py' % current_path # Target and GID options are only available on the BBP visualization clusters if arguments.input == 'target' or arguments.input == 'gid': print( 'ERROR, Target and GID options are only available on the BBP visualization clusters' ) exit(0) # Load morphology files (.H5 or .SWC) elif arguments.input == 'file': # Get the arguments string list arguments_string = arguments_parser.get_arguments_string( arguments=arguments) # NOTE: Using a morphology file, either in .H5 or .SWC formats is straightforward and # therefore the arguments will not change at all. In this case it is safe to pass the # arguments as they were received without any change. # Construct the shell command to run the workflow # -b : Blender background mode # --verbose : Turn off all the verbose messages # -- : Separate the framework arguments from those given to Blender shell_command = '%s -b --verbose 0 --python %s -- %s ' % ( arguments.blender, cli_interface, arguments_string) # Run NeuroMorphoVis from Blender in the background mode print('RUNNING: ' + shell_command) subprocess.call(shell_command, shell=True) # Load a directory morphology files (.H5 or .SWC) elif arguments.input == 'directory': # Get all the morphology files in this directory # TODO: Verify the installation of H5Py before running the workflow # TODO: Add the support to load .SWC files morphology_files = \ file_ops.get_files_in_directory(arguments.morphology_directory, '.h5') # If the directory is empty, give an error message if len(morphology_files) == 0: print( 'ERROR: The directory [%s] does NOT contain any morphology files' % arguments.morphology_directory) # Construct the commands for every individual morphology file for morphology_file in morphology_files: # Get the argument string for an individual file arguments_string = arguments_parser.get_arguments_string_for_individual_file( arguments=arguments, morphology_file=morphology_file) # Construct the shell command to run the workflow # -b : Blender background mode # --verbose : Turn off all the verbose messages # -- : Separate the framework arguments from those given to Blender shell_command = '%s -b --verbose 0 --python %s -- %s ' % ( arguments.blender, cli_interface, arguments_string) # Run NeuroMorphoVis from Blender in the background mode print('RUNNING: ' + shell_command) subprocess.call(shell_command, shell=True) else: print( 'ERROR: Input data source, use \'file, gid, target or directory\'') exit(0)
def run_cluster_neuromorphovis(arguments): """Run the NeuroMorphoVis framework on the BBP visualization cluster using SLURM. :param arguments: Command line arguments. """ # Use a specific circuit target if arguments.input == 'target': # Log print('Loading a target [%s] in circuit [%s]' % (arguments.target, arguments.blue_config)) # Ensure a valid blue config and a target if arguments.blue_config is None or arguments.target is None: print('ERROR: Empty circuit configuration file or target') exit(0) # Import brain try: import brain except ImportError: print( 'ERROR: Cannot import [brain], please load brain or install it' ) exit(0) # Open a circuit with a given blue config bbp_circuit = brain.Circuit(arguments.blue_config) # Create a GID-set and load the morphologies from these GIDs gids = bbp_circuit.gids(arguments.target) # Run the jobs on the cluster slurm.run_gid_jobs_on_cluster(arguments=arguments, gids=gids) # Use a single GID elif arguments.input == 'gid': print('Loading a gid [%s] in circuit [%s]' % (str(arguments.gid), arguments.blue_config)) # Ensure a valid blue config and a GID if arguments.blue_config is None or arguments.gid is None: print('ERROR: Empty circuit configuration file or GID') exit(0) # Run the jobs on the cluster slurm.run_gid_jobs_on_cluster(arguments=arguments, gids=[str(arguments.gid)]) # Use the morphology file (.H5 or .SWC) elif arguments.input == 'file': # Get the arguments string list arguments_string = arguments_parser.get_arguments_string( arguments=arguments) # Run the job on the cluster slurm.run_morphology_files_jobs_on_cluster( arguments=arguments, morphology_files=arguments.morphology_file) # Operate on a directory elif arguments.input == 'directory': # Get all the morphology files in this directory morphology_files = file_ops.get_files_in_directory( arguments.morphology_directory, '.h5') # Run the jobs on the cluster slurm.run_morphology_files_jobs_on_cluster( arguments=arguments, morphology_files=morphology_files) else: print('ERROR: Input data source, use [file, gid, target or directory]') exit(0)
def run_local_neuromorphovis(arguments): """Run the framework on a local node, basically your machine. :param arguments: Command line arguments. """ # Target and GID options are only available on the BBP visualization clusters if arguments.input == 'target' or arguments.input == 'gid': print( 'ERROR, Target and GID options are only available on the BBP visualization clusters' ) exit(0) # Load morphology files (.H5 or .SWC) elif arguments.input == 'file': # Get the arguments string list arguments_string = arguments_parser.get_arguments_string( arguments=arguments) # NOTE: Using a morphology file, either in .H5 or .SWC formats is straightforward and # therefore the arguments will not change at all. In this case it is safe to pass the # arguments as they were received without any change. # Construct the shell command to run the workflow shell_commands = create_shell_commands_for_local_execution( arguments, arguments_string) # Run NeuroMorphoVis from Blender in the background mode for shell_command in shell_commands: print('RUNNING: ' + shell_command) subprocess.call(shell_command, shell=True) # Load a directory morphology files (.H5 or .SWC) elif arguments.input == 'directory': # Get all the morphology files in this directory # TODO: Verify the installation of H5Py before running the workflow # TODO: Add the support to load .SWC files from a directory at the same time morphology_files = file_ops.get_files_in_directory( arguments.morphology_directory, '.h5') # If the directory is empty, give an error message if len(morphology_files) == 0: print( 'ERROR: The directory [%s] does NOT contain any morphology files' % arguments.morphology_directory) # A list of all the commands to be executed shell_commands = list() # Construct the commands for every individual morphology file for morphology_file in morphology_files: # Get the argument string for an individual file arguments_string = arguments_parser.get_arguments_string_for_individual_file( arguments=arguments, morphology_file=morphology_file) # Construct the shell command to run the workflow shell_commands.extend( create_shell_commands_for_local_execution( arguments, arguments_string)) # Run NeuroMorphoVis from Blender in the background mode for shell_command in shell_commands: # print('RUNNING: ' + shell_command) subprocess.call(shell_command, shell=True) else: print( 'ERROR: Input data source, use \'file, gid, target or directory\'') exit(0)
def run_local_neuromorphovis(arguments): """Run the framework on a local node, basically your machine. :param arguments: Command line arguments. """ # Use a specific circuit target if arguments.input == 'target': # Log print('Loading a target [%s] in circuit [%s]' % (arguments.target, arguments.blue_config)) # Ensure a valid blue config and a target if arguments.blue_config is None or arguments.target is None: print('ERROR: Empty circuit configuration file or target') exit(0) # Import BluePy try: import bluepy.v2 except ImportError: print('ERROR: Cannot import [BluePy], please install it') exit(0) from bluepy.v2 import Circuit # Loading a circuit circuit = Circuit(arguments.blue_config) # Loading the GIDs of the sample target within the circuit gids = circuit.cells.ids(arguments.target) shell_commands = list() for gid in gids: # Get the argument string for an individual file arguments_string = arguments_parser.get_arguments_string_for_individual_gid( arguments=arguments, gid=gid) # Construct the shell command to run the workflow shell_commands.extend( create_shell_commands_for_local_execution(arguments, arguments_string)) # Run NeuroMorphoVis from Blender in the background mode for shell_command in shell_commands: subprocess.call(shell_command, shell=True) # Use a single GID elif arguments.input == 'gid': print('Loading a gid [%s] in circuit [%s]' % (str(arguments.gid), arguments.blue_config)) # Ensure a valid blue config and a GID if arguments.blue_config is None or arguments.gid is None: print('ERROR: Empty circuit configuration file or GID') exit(0) # Get the argument string for an individual file arguments_string = arguments_parser.get_arguments_string_for_individual_gid( arguments=arguments, gid=arguments.gid) # Construct the shell command to run the workflow shell_commands = create_shell_commands_for_local_execution(arguments, arguments_string) # Run NeuroMorphoVis from Blender in the background mode for shell_command in shell_commands: subprocess.call(shell_command, shell=True) # Run the jobs on the cluster slurm.run_gid_jobs_on_cluster(arguments=arguments, gids=[str(arguments.gid)]) # Load morphology files (.H5 or .SWC) elif arguments.input == 'file': # Get the arguments string list arguments_string = arguments_parser.get_arguments_string(arguments=arguments) # NOTE: Using a morphology file, either in .H5 or .SWC formats is straightforward and # therefore the arguments will not change at all. In this case it is safe to pass the # arguments as they were received without any change. # Construct the shell command to run the workflow shell_commands = create_shell_commands_for_local_execution(arguments, arguments_string) # Run NeuroMorphoVis from Blender in the background mode for shell_command in shell_commands: subprocess.call(shell_command, shell=True) # Load a directory morphology files (.H5 or .SWC) elif arguments.input == 'directory': # Get all the morphology files in this directory morphology_files = file_ops.get_files_in_directory(arguments.morphology_directory, '.h5') morphology_files.extend(file_ops.get_files_in_directory( arguments.morphology_directory, '.swc')) # If the directory is empty, give an error message if len(morphology_files) == 0: print('ERROR: The directory [%s] does NOT contain any morphology files' % arguments.morphology_directory) # A list of all the commands to be executed shell_commands = list() # Construct the commands for every individual morphology file for morphology_file in morphology_files: # Get the argument string for an individual file arguments_string = arguments_parser.get_arguments_string_for_individual_file( arguments=arguments, morphology_file=morphology_file) # Construct the shell command to run the workflow shell_commands.extend( create_shell_commands_for_local_execution(arguments, arguments_string)) # Run NeuroMorphoVis from Blender in the background mode for shell_command in shell_commands: print('RUNNING: **********************************************************************') print(shell_command) print('*******************************************************************************') subprocess.call(shell_command, shell=True) else: print('ERROR: Input data source, use \'file, gid, target or directory\'') exit(0)
def submit_batch_jobs(user_name, slurm_jobs_directory): """Submits all the batch jobs found in the jobs directory. This function takes into account the maximum limit imposed by the cluster (500 jobs per user). :param user_name: The user name of the current user. :param slurm_jobs_directory: The directory where the batch jobs are created. . """ # Get all the scripts in the slurm jobs directory to submit them scripts = file_ops.get_files_in_directory(slurm_jobs_directory, file_extension='.sh') # Use an index to keep track on the number of jobs submitted to the cluster. script_index = 0 # Submit the jobs taking into account the maximum number of jobs dedicated per user while True: # Just sleep for a second time.sleep(1) # Get the number of jobs active for that user number_active_jobs = get_current_number_jobs_for_user( user_name=user_name) # If the number of jobs is greater than 500, then wait a second and try again if number_active_jobs >= 500: print('Waiting for resources ...') continue # Otherwise, you can submit some jobs else: # Get the number of jobs that are available to submit number_available_jobs = 500 - number_active_jobs # Submit as many jobs as you can for i in range(number_available_jobs): # Make sure that we still have some jobs to submit, otherwise break if script_index >= len(scripts): return # Get the script full path script_full_path = '%s/%s' % (slurm_jobs_directory, scripts[script_index]) # 'chmod' the script to be able to execute it shell_command = 'chmod +x %s' % script_full_path # Execute the command subprocess.call(shell_command, shell=True) # Format the shell command shell_command = 'sbatch %s' % script_full_path # Execute the command print('Submitting [%s]' % shell_command) subprocess.call(shell_command, shell=True) # Increment the script index script_index += 1