def submit(script, workspace, **params): """Submit a job with the given parameters.""" from klab import cluster, process # Make sure the rosetta symlink has been created. # if not os.path.exists(workspace.rosetta_dir): # raise pipeline.RosettaNotFound(workspace) # Parse some job parameters for the keyword arguments. params = dict((k, v) for k, v in list(params.items()) if v is not None) test_run = params.get('test_run', False) nstruct = params.get('nstruct') max_runtime = params.get('max_runtime', '6:00:00') max_memory = params.get('max_memory', '1G') if test_run: max_runtime = '0:30:00' if nstruct is None: raise TypeError( "sumbit() requires the keyword argument 'nstruct' for production runs." ) # Submit the job and put it immediately into the hold state. qsub_command = 'qsub', '-h', '-cwd' qsub_command += '-o', workspace.log_dir qsub_command += '-e', workspace.log_dir qsub_command += '-t', '1-{0}'.format(nstruct), qsub_command += '-l', 'h_rt={0}'.format(max_runtime), qsub_command += '-l', 'mem_free={0}'.format(max_memory), qsub_command += '-b', 'y', qsub_command += '-N', params.get('job_name'), qsub_command += workspace.python_path, qsub_command += script, qsub_command += workspace.focus_dir, status = process.check_output(qsub_command).decode('utf-8') status_pattern = re.compile( r'Your job-array (\d+).[0-9:-]+ \(".*"\) has been submitted') status_match = status_pattern.match(status) if not status_match: print(status) sys.exit() # Figure out the job id, then make a params file specifically for it. job_id = status_match.group(1) with open(workspace.job_info_path(job_id), 'w') as file: json.dump(params, file) # Release the hold on the job. qrls_command = 'qrls', job_id process.check_output(qrls_command) print(status, end=' ')
def submit(**params): from klab import cluster, process import json args = docopt.docopt(__doc__) #script_path = os.path.expanduser(args['<script>']) logdir = 'logs' csv_path = '2020-03-18_Krogan_SARSCoV2_27baits.txt' if not os.path.exists(logdir): os.makedirs(logdir, exist_ok=True) #num_tasks = file_len(csv_path) * 2 pdbs = sorted(glob.glob('pdbs/*.pdb')) num_tasks = len(pdbs)//5 + 1 max_runtime = params.get('max_runtime','24:00:00') max_memory = params.get('max_memory','4G') python = '/wynton/home/kortemme/krivacic/software/anaconda3/bin/python3' script_path = os.path.expanduser('~/ab/score_abs.py') qsub_command = 'qsub', '-h', '-cwd', qsub_command += '-b', qsub_command += 'y', qsub_command += '-o', logdir, #qsub_command += '-e', error_directory, qsub_command += '-j', 'y', qsub_command += '-t', '1-{0}'.format(num_tasks), qsub_command += '-l', 'h_rt={0}'.format(max_runtime), qsub_command += '-l', 'mem_free={0}'.format(max_memory), qsub_command += python, qsub_command += script_path, print(qsub_command) status = process.check_output(qsub_command, stderr=subprocess.STDOUT).decode('utf-8') status_pattern = re.compile(r'Your job-array (\d+).[0-9:-]+ \(".*"\) has been submitted') status_match = status_pattern.match(status) if not status_match: print(status) sys.exit() # Figure out the job id, then make a params file for it. job_id = status_match.group(1) qrls_command = 'qrls', job_id process.check_output(qrls_command) print(status)
def main(): args = docopt.docopt(__doc__) max_runtime = args['--max-runtime'] max_memory = args['--max-memory'] if args['--tasks']: ntask = args['--tasks'] else: print( glob.glob(args['<parent_directory>'] + '/*/cluster_representatives')) ntask = len( glob.glob(args['<parent_directory>'] + '/*/cluster_representatives/*turn/')) os.makedirs(args['--log'], exist_ok=True) qsub_command = 'qsub', '-h', '-cwd' qsub_command += '-o', args['--log'] qsub_command += '-e', args['--log'] qsub_command += '-t', '1-{0}'.format(ntask) qsub_command += '-l', 'h_rt={}'.format(max_runtime) qsub_command += '-l', 'mem_free={}'.format(max_memory) qsub_command += '-b', 'y' qsub_command += '-N', 'helix_matcher' qsub_command += os.environ['ROSEASY_PYTHON'], qsub_command += 'run_match.py', qsub_command += args['<parent_directory>'], qsub_command += args['--settings'], qsub_command += os.environ['ROSEASY_PYTHON'], status = process.check_output(qsub_command).decode('utf-8') status_pattern = re.compile( r'Your job-array (\d+).[0-9:-]+ \(".*"\) has been submitted') status_match = status_pattern.match(status) if not status_match: print(status) sys.exit() job_id = status_match.group(1) qrls_command = 'qrls', job_id process.check_output(qrls_command) print(status, end=' ')
def main(): args = docopt.docopt(__doc__) max_runtime = args['--max-runtime'] max_memory = args['--max-memory'] if '--tasks' in args: ntask = args['--tasks'] else: ntask = len(glob.glob(args['<pdb>'] + '/*.pdb.gz')) qsub_command = 'qsub', '-h', '-cwd' qsub_command += '-o', args['--log'] qsub_command += '-e', args['--log'] qsub_command += '-t', '1-{0}'.format(ntask) qsub_command += '-l', 'h_rt={}'.format(max_runtime) qsub_command += '-l', 'mem_free={}'.format(max_memory) qsub_command += '-b', 'y' qsub_command += '-N', 'helix_matcher' qsub_command += os.environ['ROSEASY_PYTHON'], qsub_command += 'matcher.py', qsub_command += 'match', qsub_command += args['<pdb>'], qsub_command += '-j', args['--tasks'] qsub_command += '-o', args['--out'] status = process.check_output(qsub_command).decode('utf-8') status_pattern = re.compile( r'Your job-array (\d+).[0-9:-]+ \(".*"\) has been submitted') status_match = status_pattern.match(status) if not status_match: print(status) sys.exit() job_id = status_match.group(1) qrls_command = 'qrls', job_id process.check_output(qrls_command) print(status, end=' ')
nstruct = int(find_nstruct(pdb_prefix) / num) max_runtime = '10:00:00' max_memory = '6G' logdir = 'interface_finder/logs/' os.makedirs(logdir, exist_ok=True) qsub_command = 'qsub', '-h', '-cwd' qsub_command += '-o', logdir qsub_command += '-e', logdir qsub_command += '-t', '1-{}'.format(nstruct) qsub_command += '-l', 'h_rt={}'.format(max_runtime) qsub_command += '-l', 'mem_free={}'.format(max_memory) qsub_command += '-b', 'y' qsub_command += '-N', 'helix_scan' qsub_command += '/wynton/home/kortemme/krivacic/software/anaconda3/bin/python', qsub_command += 'build_interface_list.py', str(num) status = process.check_output(qsub_command).decode('utf-8') status_pattern = re.compile( r'Your job-array (\d+).[0-9:-]+ \(".*"\) has been submitted') status_match = status_pattern.match(status) if not status_match: print(status) sys.exit() job_id = status_match.group(1) qrls_command = 'qrls', job_id process.check_output(qrls_command) print(status, end=' ')
def submit(alignments, **params): from klab import cluster, process import json args = docopt.docopt(__doc__) print(args) csv_path = args['<input_df>'] #num_tasks = (file_len(alignments) // task_len) + 1 #error_directory = 'errors' mover = args['<mover>'] logdir = os.path.join('logs', mover, 'jacobi_may_2020') if not os.path.exists(logdir): os.makedirs(logdir, exist_ok=True) outdir = args['<output_dir>'] num_tasks = file_len(csv_path) * 8 # Splitting each row into 8 #tasks, one for constrained and one for unconstrained, then those #into 2 to make the task complete on time max_runtime = params.get('max_runtime', '24:00:00') max_memory = params.get('max_memory', '4G') python = '/wynton/home/kortemme/krivacic/software/anaconda36/bin/python' script_path = os.path.expanduser( '~/intelligent_design/sidechain_directed_design_pyrosetta/inverse_rotamers/test_movers.py' ) qsub_command = 'qsub', '-h', '-cwd', qsub_command += '-b', qsub_command += 'y', qsub_command += '-o', logdir, #qsub_command += '-e', error_directory, qsub_command += '-j', 'y', qsub_command += '-t', '1-{0}'.format(num_tasks), qsub_command += '-l', 'h_rt={0}'.format(max_runtime), qsub_command += '-l', 'mem_free={0}'.format(max_memory), #qsub_command += python, qsub_command += script_path, qsub_command += csv_path, qsub_command += outdir, qsub_command += mover, qsub_command += '--br', if args['--fast']: qsub_command += '--fast', if args['--cen_temp_cycles']: qsub_command += '--cen_temp_cycles=' + args['--cen_temp_cycles'], if args['--fa_temp_cycles']: qsub_command += '--fa_temp_cycles=' + args['--fa_temp_cycles'], if args['--control']: qsub_command += '--control', print(qsub_command) status = process.check_output(qsub_command, stderr=subprocess.STDOUT).decode('utf-8') status_pattern = re.compile( r'Your job-array (\d+).[0-9:-]+ \(".*"\) has been submitted') status_match = status_pattern.match(status) if not status_match: print(status) sys.exit() # Figure out the job id, then make a params file for it. job_id = status_match.group(1) qrls_command = 'qrls', job_id process.check_output(qrls_command) print(status)
def submit_slurm(workspace, **params): from klab import process """Submit a job to a workstation with a Slurm job scheduler.""" # Parse some job parameters for the keyword arguments. params = dict((k, v) for k, v in list(params.items()) if v is not None) test_run = params.get('test_run', False) nstruct = params.get('nstruct') max_runtime = params.get('max_runtime', '12:00:00') max_memory = params.get('max_memory', '3G') if test_run: max_runtime = '0:30:00' if nstruct is None: raise TypeError( "sumbit() requires the keyword argument 'nstruct' for production runs." ) # Create the job file workspace_jobno = workspace.slurm_custom_jobno + 1 job_file = os.path.join(workspace.focus_dir, 'slurm_{}'.format(workspace_jobno)) nstruct = params.get('nstruct') f = open(job_file, 'w') for n in range(1, nstruct + 1): cmd = '{0} {1} {2} {3}'.format(workspace.python_path, workspace.script_path, workspace.focus_dir, n) f.write(cmd + '\n') f.close() submission_script = '''#!/bin/bash # # Simple SLURM script for submitting multiple serial # commands (e.g. parametric studies) using a script wrapper # to launch the commands. # # To use, change this job script to accommodate # running your serial application(s) in your WORKDIR # directory (usually the directory of submission). # Edit the commands file to specify the executions # for the launcher (paramrun) to perform. #------------------------------------------------------- #------------------------------------------------------- # # <------ Setup Parameters ------> # #SBATCH -J {name} #SBATCH -N 1 #SBATCH -n 1 #use site recommended # of cores #SBATCH -p skx-normal #SBATCH -o {logs}.o%j #SBATCH -e {logs}.e%j #SBATCH -t {runtime} ##SBATCH -A <acct_name> #uncomment and insert acct name if necessary #------------------------------------------------------ # # USING SLURM; plugins defines SLURM env. vars. export LAUNCHER_RMI=SLURM export LAUNCHER_PLUGIN_DIR=$LAUNCHER_DIR/plugins # # JOB_FILE is a list of executions to run export LAUNCHER_JOB_FILE={commands} export LAUNCHER_SCHED=dynamic export LAUNCHER_WORKDIR={focus_dir} $LAUNCHER_DIR/paramrun # will run the executions in the LAUNCHER_JOB_FILE file # "JOB" is a misnomer--these are not slurm jobs # Each line in the commands file is an # execution.'''.format( name=params.get('job_name', 'roseasy_job'), logs=os.path.join(workspace.focus_dir, 'logs', params.get('job_name', 'roseasy_job')) + '.', runtime=max_runtime, commands=job_file, focus_dir=workspace.focus_dir) with open(workspace.slurm_submit_file, 'w') as f: f.write(submission_script) status = process.check_output( ('bash', workspace.slurm_submit_file)).decode('utf-8') print('Job submission status:') print(status) #submission = with open(workspace.job_info_path(workspace_jobno), 'w') as file: json.dump(params, file)