def submit(script, workspace, **params):
"""Submit a job with the given parameters."""
from klab import cluster, process
# Make sure the rosetta symlink has been created.
# if not os.path.exists(workspace.rosetta_dir):
# raise pipeline.RosettaNotFound(workspace)
# Parse some job parameters for the keyword arguments.
params = dict((k, v) for k, v in list(params.items()) if v is not None)
test_run = params.get('test_run', False)
nstruct = params.get('nstruct')
max_runtime = params.get('max_runtime', '6:00:00')
max_memory = params.get('max_memory', '1G')
if test_run:
max_runtime = '0:30:00'
if nstruct is None:
raise TypeError(
"sumbit() requires the keyword argument 'nstruct' for production runs."
)
# Submit the job and put it immediately into the hold state.
qsub_command = 'qsub', '-h', '-cwd'
qsub_command += '-o', workspace.log_dir
qsub_command += '-e', workspace.log_dir
qsub_command += '-t', '1-{0}'.format(nstruct),
qsub_command += '-l', 'h_rt={0}'.format(max_runtime),
qsub_command += '-l', 'mem_free={0}'.format(max_memory),
qsub_command += '-b', 'y',
qsub_command += '-N', params.get('job_name'),
qsub_command += workspace.python_path,
qsub_command += script,
qsub_command += workspace.focus_dir,
status = process.check_output(qsub_command).decode('utf-8')
status_pattern = re.compile(
r'Your job-array (\d+).[0-9:-]+ \(".*"\) has been submitted')
status_match = status_pattern.match(status)
if not status_match:
print(status)
sys.exit()
# Figure out the job id, then make a params file specifically for it.
job_id = status_match.group(1)
with open(workspace.job_info_path(job_id), 'w') as file:
json.dump(params, file)
# Release the hold on the job.
qrls_command = 'qrls', job_id
process.check_output(qrls_command)
print(status, end=' ')
def submit(**params):
from klab import cluster, process
import json
args = docopt.docopt(__doc__)
#script_path = os.path.expanduser(args['<script>'])
logdir = 'logs'
csv_path = '2020-03-18_Krogan_SARSCoV2_27baits.txt'
if not os.path.exists(logdir):
os.makedirs(logdir, exist_ok=True)
#num_tasks = file_len(csv_path) * 2
pdbs = sorted(glob.glob('pdbs/*.pdb'))
num_tasks = len(pdbs)//5 + 1
max_runtime = params.get('max_runtime','24:00:00')
max_memory = params.get('max_memory','4G')
python = '/wynton/home/kortemme/krivacic/software/anaconda3/bin/python3'
script_path = os.path.expanduser('~/ab/score_abs.py')
qsub_command = 'qsub', '-h', '-cwd',
qsub_command += '-b',
qsub_command += 'y',
qsub_command += '-o', logdir,
#qsub_command += '-e', error_directory,
qsub_command += '-j', 'y',
qsub_command += '-t', '1-{0}'.format(num_tasks),
qsub_command += '-l', 'h_rt={0}'.format(max_runtime),
qsub_command += '-l', 'mem_free={0}'.format(max_memory),
qsub_command += python,
qsub_command += script_path,
print(qsub_command)
status = process.check_output(qsub_command, stderr=subprocess.STDOUT).decode('utf-8')
status_pattern = re.compile(r'Your job-array (\d+).[0-9:-]+ \(".*"\) has been submitted')
status_match = status_pattern.match(status)
if not status_match:
print(status)
sys.exit()
# Figure out the job id, then make a params file for it.
job_id = status_match.group(1)
qrls_command = 'qrls', job_id
process.check_output(qrls_command)
print(status)
def main():
args = docopt.docopt(__doc__)
max_runtime = args['--max-runtime']
max_memory = args['--max-memory']
if args['--tasks']:
ntask = args['--tasks']
else:
print(
glob.glob(args['<parent_directory>'] +
'/*/cluster_representatives'))
ntask = len(
glob.glob(args['<parent_directory>'] +
'/*/cluster_representatives/*turn/'))
os.makedirs(args['--log'], exist_ok=True)
qsub_command = 'qsub', '-h', '-cwd'
qsub_command += '-o', args['--log']
qsub_command += '-e', args['--log']
qsub_command += '-t', '1-{0}'.format(ntask)
qsub_command += '-l', 'h_rt={}'.format(max_runtime)
qsub_command += '-l', 'mem_free={}'.format(max_memory)
qsub_command += '-b', 'y'
qsub_command += '-N', 'helix_matcher'
qsub_command += os.environ['ROSEASY_PYTHON'],
qsub_command += 'run_match.py',
qsub_command += args['<parent_directory>'],
qsub_command += args['--settings'],
qsub_command += os.environ['ROSEASY_PYTHON'],
status = process.check_output(qsub_command).decode('utf-8')
status_pattern = re.compile(
r'Your job-array (\d+).[0-9:-]+ \(".*"\) has been submitted')
status_match = status_pattern.match(status)
if not status_match:
print(status)
sys.exit()
job_id = status_match.group(1)
qrls_command = 'qrls', job_id
process.check_output(qrls_command)
print(status, end=' ')
def main():
args = docopt.docopt(__doc__)
max_runtime = args['--max-runtime']
max_memory = args['--max-memory']
if '--tasks' in args:
ntask = args['--tasks']
else:
ntask = len(glob.glob(args['<pdb>'] + '/*.pdb.gz'))
qsub_command = 'qsub', '-h', '-cwd'
qsub_command += '-o', args['--log']
qsub_command += '-e', args['--log']
qsub_command += '-t', '1-{0}'.format(ntask)
qsub_command += '-l', 'h_rt={}'.format(max_runtime)
qsub_command += '-l', 'mem_free={}'.format(max_memory)
qsub_command += '-b', 'y'
qsub_command += '-N', 'helix_matcher'
qsub_command += os.environ['ROSEASY_PYTHON'],
qsub_command += 'matcher.py',
qsub_command += 'match',
qsub_command += args['<pdb>'],
qsub_command += '-j', args['--tasks']
qsub_command += '-o', args['--out']
status = process.check_output(qsub_command).decode('utf-8')
status_pattern = re.compile(
r'Your job-array (\d+).[0-9:-]+ \(".*"\) has been submitted')
status_match = status_pattern.match(status)
if not status_match:
print(status)
sys.exit()
job_id = status_match.group(1)
qrls_command = 'qrls', job_id
process.check_output(qrls_command)
print(status, end=' ')
nstruct = int(find_nstruct(pdb_prefix) / num)
max_runtime = '10:00:00'
max_memory = '6G'
logdir = 'interface_finder/logs/'
os.makedirs(logdir, exist_ok=True)
qsub_command = 'qsub', '-h', '-cwd'
qsub_command += '-o', logdir
qsub_command += '-e', logdir
qsub_command += '-t', '1-{}'.format(nstruct)
qsub_command += '-l', 'h_rt={}'.format(max_runtime)
qsub_command += '-l', 'mem_free={}'.format(max_memory)
qsub_command += '-b', 'y'
qsub_command += '-N', 'helix_scan'
qsub_command += '/wynton/home/kortemme/krivacic/software/anaconda3/bin/python',
qsub_command += 'build_interface_list.py', str(num)
status = process.check_output(qsub_command).decode('utf-8')
status_pattern = re.compile(
r'Your job-array (\d+).[0-9:-]+ \(".*"\) has been submitted')
status_match = status_pattern.match(status)
if not status_match:
print(status)
sys.exit()
job_id = status_match.group(1)
qrls_command = 'qrls', job_id
process.check_output(qrls_command)
print(status, end=' ')
def submit(alignments, **params):
from klab import cluster, process
import json
args = docopt.docopt(__doc__)
print(args)
csv_path = args['<input_df>']
#num_tasks = (file_len(alignments) // task_len) + 1
#error_directory = 'errors'
mover = args['<mover>']
logdir = os.path.join('logs', mover, 'jacobi_may_2020')
if not os.path.exists(logdir):
os.makedirs(logdir, exist_ok=True)
outdir = args['<output_dir>']
num_tasks = file_len(csv_path) * 8 # Splitting each row into 8
#tasks, one for constrained and one for unconstrained, then those
#into 2 to make the task complete on time
max_runtime = params.get('max_runtime', '24:00:00')
max_memory = params.get('max_memory', '4G')
python = '/wynton/home/kortemme/krivacic/software/anaconda36/bin/python'
script_path = os.path.expanduser(
'~/intelligent_design/sidechain_directed_design_pyrosetta/inverse_rotamers/test_movers.py'
)
qsub_command = 'qsub', '-h', '-cwd',
qsub_command += '-b',
qsub_command += 'y',
qsub_command += '-o', logdir,
#qsub_command += '-e', error_directory,
qsub_command += '-j', 'y',
qsub_command += '-t', '1-{0}'.format(num_tasks),
qsub_command += '-l', 'h_rt={0}'.format(max_runtime),
qsub_command += '-l', 'mem_free={0}'.format(max_memory),
#qsub_command += python,
qsub_command += script_path,
qsub_command += csv_path,
qsub_command += outdir,
qsub_command += mover,
qsub_command += '--br',
if args['--fast']:
qsub_command += '--fast',
if args['--cen_temp_cycles']:
qsub_command += '--cen_temp_cycles=' + args['--cen_temp_cycles'],
if args['--fa_temp_cycles']:
qsub_command += '--fa_temp_cycles=' + args['--fa_temp_cycles'],
if args['--control']:
qsub_command += '--control',
print(qsub_command)
status = process.check_output(qsub_command,
stderr=subprocess.STDOUT).decode('utf-8')
status_pattern = re.compile(
r'Your job-array (\d+).[0-9:-]+ \(".*"\) has been submitted')
status_match = status_pattern.match(status)
if not status_match:
print(status)
sys.exit()
# Figure out the job id, then make a params file for it.
job_id = status_match.group(1)
qrls_command = 'qrls', job_id
process.check_output(qrls_command)
print(status)
def submit_slurm(workspace, **params):
from klab import process
"""Submit a job to a workstation with a Slurm job scheduler."""
# Parse some job parameters for the keyword arguments.
params = dict((k, v) for k, v in list(params.items()) if v is not None)
test_run = params.get('test_run', False)
nstruct = params.get('nstruct')
max_runtime = params.get('max_runtime', '12:00:00')
max_memory = params.get('max_memory', '3G')
if test_run:
max_runtime = '0:30:00'
if nstruct is None:
raise TypeError(
"sumbit() requires the keyword argument 'nstruct' for production runs."
)
# Create the job file
workspace_jobno = workspace.slurm_custom_jobno + 1
job_file = os.path.join(workspace.focus_dir,
'slurm_{}'.format(workspace_jobno))
nstruct = params.get('nstruct')
f = open(job_file, 'w')
for n in range(1, nstruct + 1):
cmd = '{0} {1} {2} {3}'.format(workspace.python_path,
workspace.script_path,
workspace.focus_dir, n)
f.write(cmd + '\n')
f.close()
submission_script = '''#!/bin/bash
#
# Simple SLURM script for submitting multiple serial
# commands (e.g. parametric studies) using a script wrapper
# to launch the commands.
#
# To use, change this job script to accommodate
# running your serial application(s) in your WORKDIR
# directory (usually the directory of submission).
# Edit the commands file to specify the executions
# for the launcher (paramrun) to perform.
#-------------------------------------------------------
#-------------------------------------------------------
#
# <------ Setup Parameters ------>
#
#SBATCH -J {name}
#SBATCH -N 1
#SBATCH -n 1 #use site recommended # of cores
#SBATCH -p skx-normal
#SBATCH -o {logs}.o%j
#SBATCH -e {logs}.e%j
#SBATCH -t {runtime}
##SBATCH -A <acct_name> #uncomment and insert acct name if necessary
#------------------------------------------------------
# # USING SLURM; plugins defines SLURM env. vars.
export LAUNCHER_RMI=SLURM
export LAUNCHER_PLUGIN_DIR=$LAUNCHER_DIR/plugins
# # JOB_FILE is a list of executions to run
export LAUNCHER_JOB_FILE={commands}
export LAUNCHER_SCHED=dynamic
export LAUNCHER_WORKDIR={focus_dir}
$LAUNCHER_DIR/paramrun # will run the executions in the LAUNCHER_JOB_FILE file
# "JOB" is a misnomer--these are not slurm jobs
# Each line in the commands file is an
# execution.'''.format(
name=params.get('job_name', 'roseasy_job'),
logs=os.path.join(workspace.focus_dir, 'logs',
params.get('job_name', 'roseasy_job')) + '.',
runtime=max_runtime,
commands=job_file,
focus_dir=workspace.focus_dir)
with open(workspace.slurm_submit_file, 'w') as f:
f.write(submission_script)
status = process.check_output(
('bash', workspace.slurm_submit_file)).decode('utf-8')
print('Job submission status:')
print(status)
#submission =
with open(workspace.job_info_path(workspace_jobno), 'w') as file:
json.dump(params, file)
