def submit(self, workflow):
""" Submits workflow
"""
# create scratch directories
unix.mkdir(PATH.SCRATCH)
unix.mkdir(PATH.SYSTEM)
# create output directories
unix.mkdir(PATH.OUTPUT)
unix.mkdir(PATH.WORKDIR+'/'+'output.pbs')
workflow.checkpoint()
hours = PAR.WALLTIME/60
minutes = PAR.WALLTIME%60
walltime = 'walltime=%02d:%02d:00 ' % (hours, minutes)
ncpus = PAR.NODESIZE
mpiprocs = PAR.NODESIZE
# prepare qsub arguments
call( 'qsub '
+ '%s ' % PAR.PBSARGS
+ '-l select=1:ncpus=%d:mpiprocs=%d ' % (ncpus, mpiprocs)
+ '-l %s ' % walltime
+ '-N %s ' % PAR.TITLE
+ '-j %s '%'oe'
+ '-o %s ' % (PATH.WORKDIR+'/'+'output.log')
+ '-V '
+ ' -- ' + findpath('seisflows.system') +'/'+ 'wrappers/submit '
+ PATH.OUTPUT)
def submit(self, workflow):
""" Submits workflow
"""
# create scratch directories
unix.mkdir(PATH.SCRATCH)
unix.mkdir(PATH.SYSTEM)
# create output directories
unix.mkdir(PATH.OUTPUT)
unix.mkdir(PATH.WORKDIR+'/'+'output.pbs')
self.checkpoint()
hours = PAR.WALLTIME/60
minutes = PAR.WALLTIME%60
walltime = 'walltime=%02d:%02d:00 ' % (hours, minutes)
ncpus = PAR.NODESIZE
mpiprocs = PAR.NODESIZE
# prepare qsub arguments
call( 'qsub '
+ '%s ' % PAR.PBSARGS
+ '-l select=1:ncpus=%d:mpiprocs=%d ' % (ncpus, mpiprocs)
+ '-l %s ' % walltime
+ '-N %s ' % PAR.TITLE
+ '-j %s '%'oe'
+ '-o %s ' % (PATH.WORKDIR+'/'+'output.log')
+ '-V '
+ ' -- ' + findpath('seisflows.system') +'/'+ 'wrappers/submit '
+ PATH.OUTPUT)
def run(self, classname, funcname, hosts='all', **kwargs):
""" Runs task(s) on specified hosts
"""
self.checkpoint()
if hosts == 'all':
# run all tasks
call(findpath('seisflows.system') +'/'+'wrappers/dsh '
+ ','.join(self.hostlist()) + ' '
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname + ' '
+ findpath('seisflows.system') +'/'+'wrappers/run '
+ 'PYTHONPATH='+findpath('seisflows.system'),+','
+ PAR.ENVIRONS)
elif hosts == 'head':
# run a single task
call('ssh ' + self.hostlist()[0] + ' '
+ '"'
+ 'export SEISFLOWS_TASK_ID=0; '
+ join(findpath('seisflows.system'), 'wrappers/run ')
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname + ' '
+ 'PYTHONPATH='+findpath('seisflows.system'),+','
+ PAR.ENVIRONS
+'"')
else:
raise KeyError('Bad keyword argument: system.run: hosts')
def submit(self, *args, **kwargs):
""" Submits job
"""
# create scratch directories
if not exists(PATH.SCRATCH):
path = '/scratch/gpfs' + '/' + getuser(
) + '/' + 'seisflows' + '/' + str(uuid4())
unix.mkdir(path)
unix.ln(path, PATH.SCRATCH)
unix.mkdir(PATH.SYSTEM)
# create output directories
unix.mkdir(PATH.OUTPUT)
unix.mkdir(PATH.WORKDIR + '/' + 'output.slurm')
self.checkpoint()
if not exists(PATH.SCRATCH):
path = '/scratch/gpfs' + '/' + getuser(
) + '/' + 'seisflows' + '/' + str(uuid4())
unix.mkdir(path)
unix.ln(path, PATH.SCRATCH)
call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s ' % PAR.TITLE +
'--output=%s ' % (PATH.WORKDIR + '/' + 'output.log') +
'--ntasks-per-node=%d ' % 28 + '--gres=gpu:%d ' % 4 +
'--nodes=%d ' % 1 + '--time=%d ' % PAR.WALLTIME +
pkgpath('seisflows') + '/' + 'system/wrappers/submit ' +
PATH.OUTPUT)
def run(self, classname, funcname, hosts='all', **kwargs):
""" Runs tasks in serial or parallel on specified hosts
"""
self.checkpoint()
self.save_kwargs(classname, funcname, kwargs)
if hosts == 'all':
call(findpath('seisflows.system') +'/'+'wrappers/dsh '
+ str(PAR.NTASK) + ' '
+ findpath('seisflows.system') +'/'+'wrappers/run '
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname + ' '
+ PAR.ENVIRONS)
#call('srun '
# + '--wait=0 '
# + '--exclusive '
# + join(findpath('seisflows.system'), 'wrappers/run ')
# + PATH.OUTPUT + ' '
# + classname + ' '
# + funcname + ' '
# + PAR.ENVIRONS)
elif hosts == 'head':
# run on head node
call(findpath('seisflows.system') +'/'+'wrappers/run '
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname + ' '
+ PAR.ENVIRONS)
def run(self, classname, funcname, hosts='all', **kwargs):
""" Runs tasks in serial or parallel on specified hosts
"""
self.checkpoint()
self.save_kwargs(classname, funcname, kwargs)
if hosts == 'all':
# run on all available nodes
call('srun '
+ '--wait=0 '
+ join(findpath('seisflows.system'), 'wrappers/run ')
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname + ' '
+ PAR.ENVIRONS)
elif hosts == 'head':
# run on head node
call('srun '
+ '--wait=0 '
+ '--ntasks=1 '
+ '--nodes=1 '
+ join(findpath('seisflows.system'), 'wrappers/run ')
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname + ' '
+ PAR.ENVIRONS)
else:
raise KeyError('Bad keyword argument: system.run: hosts')
def run(self, classname, method, hosts='all', **kwargs):
""" Executes the following task:
classname.method(*args, **kwargs)
"""
self.checkpoint()
self.save_kwargs(classname, method, kwargs)
if hosts == 'all':
# run on all available nodes
call(join(pkgpath('seisflows-hpc'), 'system/wrappers/dsh_tigergpu') + ' '
+ join(pkgpath('seisflows'), 'system/wrappers/run') + ' '
+ PATH.OUTPUT + ' '
+ classname + ' '
+ method + ' '
+ str(PAR.NTASK) + ' '
+ PAR.ENVIRONS)
elif hosts == 'head':
# run on head node
call('srun '
+ '--ntasks=1 '
+ '--nodes=1 '
+ join(pkgpath('seisflows'), 'system/wrappers/run')
+ PATH.OUTPUT + ' '
+ classname + ' '
+ method + ' '
+ PAR.ENVIRONS)
def run(self, classname, method, hosts='all', **kwargs):
""" Executes the following task:
classname.method(*args, **kwargs)
"""
self.checkpoint()
if hosts == 'all':
# run all tasks
call(
findpath('seisflows.system') + '/' + 'wrappers/dsh ' +
','.join(self.hostlist()) + ' ' +
findpath('seisflows.system') + '/' + 'wrappers/run ' +
PATH.OUTPUT + ' ' + classname + ' ' + method + ' ' +
'PYTHONPATH=' + findpath('seisflows'), + ',' + PAR.ENVIRONS)
elif hosts == 'head':
# run a single task
call(
'ssh ' + self.hostlist()[0] + ' ' + '"' +
'export SEISFLOWS_TASK_ID=0; ' +
join(findpath('seisflows.system'), 'wrappers/run ') +
PATH.OUTPUT + ' ' + classname + ' ' + method + ' ' +
'PYTHONPATH=' + findpath('seisflows'),
+ ',' + PAR.ENVIRONS + '"')
else:
raise KeyError('Bad keyword argument: system.run: hosts')
def submit(self, workflow):
""" Submits workflow
"""
# create scratch directories
if not exists(PATH.SCRATCH):
path = '/scratch/gpfs'+'/'+getuser()+'/'+'seisflows'+'/'+str(uuid4())
unix.mkdir(path)
unix.ln(path, PATH.SCRATCH)
unix.mkdir(PATH.SYSTEM)
# create output directories
unix.mkdir(PATH.OUTPUT)
unix.mkdir(PATH.WORKDIR+'/'+'output.slurm')
self.checkpoint()
# prepare sbatch arguments
call('sbatch '
+ '%s ' % PAR.SLURMARGS
+ '--job-name=%s ' % PAR.TITLE
+ '--output %s ' % (PATH.WORKDIR+'/'+'output.log')
+ '--ntasks-per-node=28 '
+ '--ntasks=28 '
+ '--gres=gpu:4 '
+ '--nodes=%d ' % 1
+ '--time=%d ' % PAR.WALLTIME
+ findpath('seisflows.system') +'/'+ 'wrappers/submit '
+ PATH.OUTPUT)
def run(self, classname, method, hosts='all', **kwargs):
""" Runs embarrassingly parallel tasks
Executes the following multiple times:
classname.method(*args, **kwargs)
system.taskid serves to provide each running task a unique identifier
"""
self.checkpoint(PATH.OUTPUT, classname, method, args, kwargs)
if hosts == 'all':
# run all tasks
call(
findpath('seisflows.system') + '/' + 'wrappers/dsh ' +
','.join(self.hostlist()) + ' ' +
findpath('seisflows.system') + '/' + 'wrappers/run ' +
PATH.OUTPUT + ' ' + classname + ' ' + method + ' ' +
'PYTHONPATH=' + findpath('seisflows'), + ',' + PAR.ENVIRONS)
elif hosts == 'head':
# run a single task
call(
'ssh ' + self.hostlist()[0] + ' ' + '"' +
'export SEISFLOWS_TASK_ID=0; ' +
join(findpath('seisflows.system'), 'wrappers/run ') +
PATH.OUTPUT + ' ' + classname + ' ' + method + ' ' +
'PYTHONPATH=' + findpath('seisflows'),
+ ',' + PAR.ENVIRONS + '"')
else:
raise KeyError('Bad keyword argument: system.run: hosts')
def run_single(self, classname, method, *args, **kwargs):
""" Runs task a single time
"""
self.checkpoint(PATH.OUTPUT, classname, method, args, kwargs)
call('srun ' + '--wait=0 ' + '--ntasks=1 ' + '--nodes=1 ' +
'%s ' % join(findpath('seisflows.system'), 'wrappers/run ') +
'%s ' % PATH.OUTPUT + '%s ' % classname + '%s ' % method +
'%s ' % PAR.ENVIRONS)
def run(self, classname, method, *args, **kwargs):
""" Runs task multiple times in embarrassingly parallel fasion
"""
self.checkpoint(PATH.OUTPUT, classname, method, args, kwargs)
call('srun ' + '--wait=0 ' +
'%s ' % join(findpath('seisflows.system'), 'wrappers/run ') +
'%s ' % PATH.OUTPUT + '%s ' % classname + '%s ' % method +
'%s ' % PAR.ENVIRONS)
def _query(self, jobid):
""" Retrives job state from LSF database
"""
with open(PATH.SYSTEM+'/'+'job_status', 'w') as f:
call('bjobs -a -d "' + jobid + '"', stdout=f)
with open(PATH.SYSTEM+'/'+'job_status', 'r') as f:
lines = f.readlines()
state = lines[1].split()[2].strip()
return state
def _query(self, jobid):
""" Retrives job state from LSF database
"""
with open(PATH.SYSTEM+'/'+'job_status', 'w') as f:
call('bjobs -a -d "' + jobid + '"', stdout=f)
with open(PATH.SYSTEM+'/'+'job_status', 'r') as f:
lines = f.readlines()
state = lines[1].split()[2].strip()
return state
def _query(self, jobid):
""" Queries job state from SLURM database
"""
with open(PATH.SYSTEM + '/' + 'job_status', 'w') as f:
call('sacct -n -o state -j ' + jobid, stdout=f)
with open(PATH.SYSTEM + '/' + 'job_status', 'r') as f:
line = f.readline()
state = line.strip()
return state
def _query(self, job):
""" Queries job state from SLURM database
"""
with open(PATH.SYSTEM + '/' + 'job_status', 'w') as file:
call('sacct -n -o jobid,state -j ' + job.split('_')[0],
stdout=file)
state = ''
with open(PATH.SYSTEM + '/' + 'job_status', 'r') as file:
for line in file.readlines():
if line.split()[0] == job:
state = line.split()[1]
return state
def run(self, classname, method, *args, **kwargs):
""" Runs task multiple times in embarrassingly parallel fasion
"""
self.checkpoint(PATH.OUTPUT, classname, method, args, kwargs)
call('srun '
+ '--wait=0 '
+ '%s ' % join(findpath('seisflows.system'), 'wrappers/run ')
+ '%s ' % PATH.OUTPUT
+ '%s ' % classname
+ '%s ' % method
+ '%s ' % PAR.ENVIRONS)
def submit_job_array(self, classname, funcname, hosts='all'):
with open(PATH.SYSTEM + '/' + 'job_id', 'w') as f:
call(self.job_array_cmd(classname, funcname, hosts), stdout=f)
# retrieve job ids
with open(PATH.SYSTEM + '/' + 'job_id', 'r') as f:
line = f.readline()
job = line.split()[-1].strip()
if hosts == 'all' and PAR.NTASK > 1:
return [job + '_' + str(ii) for ii in range(PAR.NTASK)]
else:
return [job]
def resubmit_failed_job(self, classname, method, jobs, taskid):
with open(PATH.SYSTEM + '/' + 'job_id', 'w') as file:
call(self.resubmit_cmd(classname, method, taskid), stdout=file)
with open(PATH.SYSTEM + '/' + 'job_id', 'r') as file:
line = file.readline()
jobid = line.split()[-1].strip()
# remove failed job from list
jobs.pop(taskid)
# add resubmitted job to list
jobs.insert(taskid, jobid)
return jobs
def submit_job_array(self, classname, funcname, hosts='all'):
with open(PATH.SYSTEM + '/' + 'array_id', 'w') as file:
call(self.job_array_cmd(classname, funcname, hosts), stdout=file)
with open(PATH.SYSTEM + '/' + 'array_id', 'r') as file:
line = file.readline()
id = line.split()[-1].strip()
if hosts == 'all':
tasks = range(PAR.NTASK)
jobs = [id + '_' + str(task) for task in tasks]
else:
jobs = [id + '_0']
return jobs
def _query(self, jobid):
""" Queries job state from PBS database
"""
# TODO: replace shell utilities with native Python
with open(PATH.SYSTEM + '/' + 'job_status', 'w') as f:
call('qstat -x -tJ ' + jobid + ' | ' + 'tail -n 1 ' + ' | ' +
'awk \'{print $5}\'',
stdout=f)
with open(PATH.SYSTEM + '/' + 'job_status', 'r') as f:
line = f.readline()
state = line.strip()
return state
def submit_job_array(self, classname, method, hosts='all'):
with open(PATH.SYSTEM + '/' + 'job_id', 'w') as f:
call(self.job_array_cmd(classname, method, hosts), stdout=f)
# retrieve job ids
with open(PATH.SYSTEM + '/' + 'job_id', 'r') as f:
line = f.readline()
job = line.split()[-1].strip()
if hosts == 'all' and PAR.NTASK > 1:
nn = range(PAR.NTASK)
job0 = job.strip('[].sdb')
return [job0 + '[' + str(ii) + '].sdb' for ii in nn]
else:
return [job]
def run_single(self, classname, method, *args, **kwargs):
""" Runs task a single time
"""
self.checkpoint(PATH.OUTPUT, classname, method, args, kwargs)
call('srun '
+ '--wait=0 '
+ '--ntasks=1 '
+ '--nodes=1 '
+ '%s ' % join(findpath('seisflows.system'), 'wrappers/run ')
+ '%s ' % PATH.OUTPUT
+ '%s ' % classname
+ '%s ' % method
+ '%s ' % PAR.ENVIRONS)
def submit_job_array(self, classname, method, hosts='all'):
with open(PATH.SYSTEM+'/'+'job_id', 'w') as f:
call(self.job_array_cmd(classname, method, hosts),
stdout=f)
# retrieve job ids
with open(PATH.SYSTEM+'/'+'job_id', 'r') as f:
line = f.readline()
job = line.split()[-1].strip()
if hosts == 'all' and PAR.NTASK > 1:
nn = range(PAR.NTASK)
job0 = job.strip('[].sdb')
return [job0+'['+str(ii)+'].sdb' for ii in nn]
else:
return [job]
def _query(self, jobid):
""" Queries job state from PBS database
"""
# TODO: replace shell utilities with native Python
with open(PATH.SYSTEM+'/'+'job_status', 'w') as f:
call('qstat -x -tJ ' + jobid + ' | '
+ 'tail -n 1 ' + ' | '
+ 'awk \'{print $5}\'',
stdout=f)
with open(PATH.SYSTEM+'/'+'job_status', 'r') as f:
line = f.readline()
state = line.strip()
return state
def submit(self, workflow):
""" Submits workflow
"""
unix.cd(PATH.WORKDIR)
if not exists('./scratch'):
unix.ln(PATH.SCRATCH, PATH.WORKDIR + '/' + 'scratch')
unix.mkdir(PATH.OUTPUT)
workflow.checkpoint()
# prepare sbatch arguments
call('sbatch ' + '%s ' % PAR.SLURMARGS +
'--partition=%s ' % PAR.PARTITION + '--job-name=%s ' % PAR.TITLE +
'--output %s ' % (PATH.WORKDIR + '/' + 'output.log') +
'--cpus-per-task=%d ' % PAR.NPROC + '--ntasks=%d ' % PAR.NTASK +
'--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') + '/' +
'wrappers/submit ' + PATH.OUTPUT)
def submit_job_array(self, classname, funcname, hosts='all'):
# submit job
with open(PATH.SYSTEM+'/'+'job_id', 'w') as f:
call(self.job_array_cmd(classname, funcname, hosts), stdout=f)
# retrieve job ids
with open(PATH.SYSTEM+'/'+'job_id', 'r') as f:
# reads one entire line from the file
line = f.readline()
job_buf = line.split()[1].strip()
job = job_buf[1:-1]
if hosts == 'all' and PAR.NSRC > 1:
nn = range(1,PAR.NSRC+1)
#return [job+'_'+str(ii) for ii in nn]
return [job+'['+str(ii)+']' for ii in nn]
else:
return [job]
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
self.checkpoint()
# if not exists(PATH.SUBMIT + '/' + 'scratch'):
# unix.ln(PATH.SCRATCH, PATH.SUBMIT + '/' + 'scratch')
nnodes = 1 #PAR.NTASK / PAR.NGPU
call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s ' % PAR.TITLE +
'--output=%s ' % (PATH.WORKDIR + '/' + 'output.log') +
'--nodes %d ' % nnodes + '--ntasks-per-node=%d ' % PAR.NGPU +
'--ntasks-per-socket=%d ' % PAR.NGPU +
'--gres=gpu:%d ' % PAR.NGPU + '--time=%d ' % PAR.WALLTIME +
findpath('seisflows.system') + '/' + 'wrappers/submit ' +
PATH.OUTPUT)
def submit(self, workflow):
""" Submits workflow
"""
# create scratch directories
unix.mkdir(PATH.SCRATCH)
unix.mkdir(PATH.SYSTEM)
# create output directories
unix.mkdir(PATH.OUTPUT)
self.checkpoint()
# submit workflow
call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s ' % PAR.TITLE +
'--output=%s ' % (PATH.WORKDIR + '/' + 'output.log') +
'--cpus-per-task=%d ' % PAR.NPROC + '--ntasks=%d ' % PAR.NTASK +
'--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') + '/' +
'wrappers/submit ' + PATH.OUTPUT)
def submit(self, workflow):
""" Submits workflow
"""
# create scratch directories
unix.mkdir(PATH.SCRATCH)
unix.mkdir(PATH.SYSTEM)
# create output directories
unix.mkdir(PATH.OUTPUT)
unix.mkdir(PATH.WORKDIR + '/' + 'output.slurm')
workflow.checkpoint()
# prepare sbatch arguments
call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s ' % PAR.TITLE +
'--output %s ' % (PATH.WORKDIR + '/' + 'output.log') +
'--ntasks-per-node=%d ' % PAR.NODESIZE + '--nodes=%d ' % 1 +
'--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') + '/' +
'wrappers/submit ' + PATH.OUTPUT)
def submit(self, workflow):
""" Submits workflow
"""
# create scratch directories
unix.mkdir(PATH.SCRATCH)
unix.mkdir(PATH.SYSTEM)
# create output directories
unix.mkdir(PATH.OUTPUT)
unix.mkdir(PATH.WORKDIR + '/' + 'output.lsf')
self.checkpoint()
# prepare bsub arguments
call('bsub ' + '%s ' % PAR.LSFARGS + '-J %s ' % PAR.TITLE + '-o %s ' %
(PATH.WORKDIR + '/' + 'output.log') + '-n %d ' % PAR.NODESIZE +
'-e %s ' % (PATH.WORKDIR + '/' + 'error.log') +
'-R "span[ptile=%d]" ' % PAR.NODESIZE +
'-W %d:00 ' % PAR.WALLTIME + findpath('seisflows.system') + '/' +
'wrappers/submit ' + PATH.OUTPUT)
def submit(self, workflow):
""" Submits workflow
"""
# create scratch directories
unix.mkdir(PATH.SCRATCH)
unix.mkdir(PATH.SYSTEM)
# create output directories
unix.mkdir(PATH.OUTPUT)
workflow.checkpoint()
# submit workflow
call('sbatch '
+ '%s ' % PAR.SLURMARGS
+ '--job-name=%s '%PAR.TITLE
+ '--output=%s '%(PATH.WORKDIR +'/'+ 'output.log')
+ '--cpus-per-task=%d '%PAR.NPROC
+ '--ntasks=%d '%PAR.NTASK
+ '--time=%d '%PAR.WALLTIME
+ '%s ' % findpath('seisflows.system') +'/'+ 'wrappers/submit'
+ '%s ' % PATH.OUTPUT)
def detect_edges(self):
path_input = PATH.MUMFORD_SHAH_INPUT
path_output = PATH.MUMFORD_SHAH_OUTPUT
path_run = PATH.SUBMIT
unix.cp(glob(PATH.GRAD+'/'+'model/*'), path_input)
unix.mkdir(path_output)
unix.cd(path_run)
# writes damping term to disk
with open('mumford_shah.log', 'w') as fileobj:
call('srun -n 1 -N 1 ' +PATH.MUMFORD_SHAH_BIN+'/'+'psemimage ' +
PATH.MUMFORD_SHAH_CONFIG +
' -ksp_type fgmres ' +
' -pc_type asm ' +
' -ksp_gmres_restart 300 ',
shell=True,
stdout=fileobj)
for parameter in solver.parameters:
self.apply_smoothing(parameter)
self.write_damping_term(parameter)
def submit(self, workflow):
""" Submits workflow
"""
# create scratch directories
unix.mkdir(PATH.SCRATCH)
unix.mkdir(PATH.SYSTEM)
# create output directories
unix.mkdir(PATH.OUTPUT)
unix.mkdir(PATH.WORKDIR+'/'+'output.slurm')
workflow.checkpoint()
# prepare sbatch arguments
call('sbatch '
+ '%s ' % PAR.SLURMARGS
+ '--job-name=%s ' % PAR.TITLE
+ '--output %s ' % (PATH.WORKDIR+'/'+'output.log')
+ '--ntasks-per-node=%d ' % PAR.NODESIZE
+ '--nodes=%d ' % 1
+ '--time=%d ' % PAR.WALLTIME
+ findpath('seisflows.system') +'/'+ 'wrappers/submit '
+ PATH.OUTPUT)
def run(self, classname, method, hosts='all', **kwargs):
""" Executes the following task:
classname.method(*args, **kwargs)
"""
self.checkpoint()
self.save_kwargs(classname, method, kwargs)
if hosts == 'all':
# run on all available nodes
call('srun ' + '--wait=0 ' +
join(findpath('seisflows.system'), 'wrappers/run ') +
PATH.OUTPUT + ' ' + classname + ' ' + method + ' ' +
PAR.ENVIRONS)
elif hosts == 'head':
# run on head node
call('srun ' + '--wait=0 ' + '--ntasks=1 ' + '--nodes=1 ' +
join(findpath('seisflows.system'), 'wrappers/run ') +
PATH.OUTPUT + ' ' + classname + ' ' + method + ' ' +
PAR.ENVIRONS)
else:
raise KeyError('Bad keyword argument: system.run: hosts')
def submit(self, workflow):
""" Submits workflow
"""
# create scratch directories
unix.mkdir(PATH.SCRATCH)
unix.mkdir(PATH.SYSTEM)
# create output directories
unix.mkdir(PATH.OUTPUT)
self.checkpoint()
# submit workflow
call('sbatch '
+ '%s ' % PAR.SLURMARGS
+ '--job-name=%s '%PAR.TITLE
+ '--output=%s '%(PATH.WORKDIR +'/'+ 'output.log')
+ '--ntasks=%d '%PAR.NTASK
+ '--nodes=%d '%ceil(PAR.NTASK/4.)
+ '--ntasks-per-node=%d '%min(PAR.NTASK,4)
+ '--gres=gpu:%d '%min(PAR.NTASK,4)
+ '--time=%d '%PAR.WALLTIME
+ pkgpath('seisflows') +'/'+ 'system/wrappers/submit '
+ PATH.OUTPUT)
def submit(self, workflow):
""" Submits workflow
"""
# create scratch directories
unix.mkdir(PATH.SCRATCH)
unix.mkdir(PATH.SYSTEM)
# create output directories
unix.mkdir(PATH.OUTPUT)
unix.mkdir(PATH.WORKDIR+'/'+'output.lsf')
workflow.checkpoint()
# prepare bsub arguments
call('bsub '
+ '%s ' % PAR.LSFARGS
+ '-J %s ' % PAR.TITLE
+ '-o %s ' % (PATH.WORKDIR+'/'+'output.log')
+ '-n %d ' % PAR.NODESIZE
+ '-e %s ' % (PATH.WORKDIR+'/'+'error.log')
+ '-R "span[ptile=%d]" ' % PAR.NODESIZE
+ '-W %d:00 ' % PAR.WALLTIME
+ findpath('seisflows.system') +'/'+ 'wrappers/submit '
+ PATH.OUTPUT)
def run(self, classname, method, hosts='all', **kwargs):
""" Runs embarrassingly parallel tasks
Executes the following multiple times:
classname.method(*args, **kwargs)
system.taskid serves to provide each running task a unique identifier
"""
self.checkpoint(PATH.OUTPUT, classname, method, args, kwargs)
if hosts == 'all':
# run all tasks
call(findpath('seisflows.system') +'/'+'wrappers/dsh '
+ ','.join(self.hostlist()) + ' '
+ findpath('seisflows.system') +'/'+'wrappers/run '
+ PATH.OUTPUT + ' '
+ classname + ' '
+ method + ' '
+ 'PYTHONPATH='+findpath('seisflows'),+','
+ PAR.ENVIRONS)
elif hosts == 'head':
# run a single task
call('ssh ' + self.hostlist()[0] + ' '
+ '"'
+ 'export SEISFLOWS_TASK_ID=0; '
+ join(findpath('seisflows.system'), 'wrappers/run ')
+ PATH.OUTPUT + ' '
+ classname + ' '
+ method + ' '
+ 'PYTHONPATH='+findpath('seisflows'),+','
+ PAR.ENVIRONS
+'"')
else:
raise KeyError('Bad keyword argument: system.run: hosts')
def hostlist(self):
with open(PATH.SYSTEM+'/'+'hostlist', 'w') as f:
call('scontrol show hostname $SLURM_JOB_NODEFILE', stdout=f)
with open(PATH.SYSTEM+'/'+'hostlist', 'r') as f:
return [line.strip() for line in f.readlines()]
def resubmit_failed_job(self, job):
call('scontrol requeue ' + job)
