def kwBlockOnShellCmd_rs(self, **kwargs):
'''
A 'kwargs' version of the 'blockOnShellCmd' call. Useful for cases
when more complex blocking conditions need to be evaluated.
The _rs denotes that this method is specialized to flagging
running *and* scheduled jobs.
This particular method is MOSIX specific.
kwargs:
blockProcess process in scheduler to block on
blockMsg log message when block starts
loopMsg log message while blocking
timeout how long between checking astr_shellCmd
while blocking
'''
for key, val in kwargs.iteritems():
if key == 'blockProcess': astr_blockProcess = val
if key == 'blockMsg': astr_blockMsg = val
if key == 'loopMsg': astr_loopMsg = val
if key == 'timeout': atimeout = val
shellScheduleCount = crun.crun()
shellRunCount = crun.crun()
astr_processInSchedulerCount = 'mosq listall | grep %s | wc -l' % astr_blockProcess
astr_allJobsDoneCount = '0'
astr_processRunningCount = 'mosq listall | grep %s | grep RUN | wc -l' % astr_blockProcess
shellScheduleCount(astr_processInSchedulerCount)
blockLoop = 1
if shellScheduleCount.stdout().strip() != astr_allJobsDoneCount:
self._log(Colors.CYAN + astr_blockMsg + Colors.NO_COLOUR)
while 1:
time.sleep(atimeout)
shellScheduleCount(astr_processInSchedulerCount)
shellRunCount(astr_processRunningCount)
str_scheduled = shellScheduleCount.stdout().strip()
str_running = shellRunCount.stdout().strip()
if str_scheduled == astr_allJobsDoneCount:
self._log('\n', syslog=False)
break
else:
str_loopMsg = Colors.BROWN + '(block duration = %ds; running/scheduled = %s/%s) '% \
(blockLoop * atimeout, str_running, str_scheduled) + \
Colors.YELLOW + astr_loopMsg + Colors.NO_COLOUR
self._log(str_loopMsg)
loopMsgLen = len(str_loopMsg)
syslogLen = len(self._log.str_syslog())
for i in range(0, loopMsgLen + syslogLen):
self._log('\b', syslog=False)
blockLoop += 1
return True
def kwBlockOnShellCmd_rs(self, **kwargs):
'''
A 'kwargs' version of the 'blockOnShellCmd' call. Useful for cases
when more complex blocking conditions need to be evaluated.
The _rs denotes that this method is specialized to flagging
running *and* scheduled jobs.
This particular method is MOSIX specific.
kwargs:
blockProcess process in scheduler to block on
blockMsg log message when block starts
loopMsg log message while blocking
timeout how long between checking astr_shellCmd
while blocking
'''
for key, val in kwargs.iteritems():
if key == 'blockProcess': astr_blockProcess = val
if key == 'blockMsg': astr_blockMsg = val
if key == 'loopMsg': astr_loopMsg = val
if key == 'timeout': atimeout = val
shellScheduleCount = crun.crun()
shellRunCount = crun.crun()
astr_processInSchedulerCount = 'mosq listall | grep %s | wc -l' % astr_blockProcess
astr_allJobsDoneCount = '0'
astr_processRunningCount = 'mosq listall | grep %s | grep RUN | wc -l' % astr_blockProcess
shellScheduleCount(astr_processInSchedulerCount)
blockLoop = 1
if shellScheduleCount.stdout().strip() != astr_allJobsDoneCount:
self._log(Colors.CYAN + astr_blockMsg + Colors.NO_COLOUR)
while 1:
time.sleep(atimeout)
shellScheduleCount(astr_processInSchedulerCount)
shellRunCount(astr_processRunningCount)
str_scheduled = shellScheduleCount.stdout().strip()
str_running = shellRunCount.stdout().strip()
if str_scheduled == astr_allJobsDoneCount:
self._log('\n', syslog=False)
break
else:
str_loopMsg = Colors.BROWN + '(block duration = %ds; running/scheduled = %s/%s) '% \
(blockLoop * atimeout, str_running, str_scheduled) + \
Colors.YELLOW + astr_loopMsg + Colors.NO_COLOUR
self._log(str_loopMsg)
loopMsgLen = len(str_loopMsg)
syslogLen = len(self._log.str_syslog())
for i in range(0, loopMsgLen+syslogLen): self._log('\b', syslog=False)
blockLoop += 1
return True
def f_stage0callback(**kwargs):
str_cwd = os.getcwd()
lst_subj = []
for key, val in kwargs.iteritems():
if key == 'subj': lst_subj = val
if key == 'obj': stage = val
for subj in lst_subj:
# find the relevant input files in each <subj> dir
os.chdir(subj)
l_B0 = misc.find(_str_b0)
if not l_B0: error.fatal(pipe_hypothermia, 'noB0')
_str_B0File = l_B0[0]
_str_outDir = '%s/outDir' % os.getcwd()
_str_outFile = 'b0Brain.nii'
misc.mkdir(_str_outDir)
str_prefixCmd = '( cd %s ; ' % (os.getcwd())
log = stage.log()
log('Scheduling brain extraction for "%s"...\n' % (subj))
str_cmd = 'bet.py --input %s --output %s/%s' % (_str_B0File,
_str_outDir, _str_outFile)
#cluster = crun.crun_mosix(cmdPrefix=str_prefixCmd)
#cluster = crun.crun_mosix()
cluster = crun.crun()
#str_ccmd = cluster.echo(True)
#log(str_ccmd)
cluster.echo(False)
cluster.echoStdOut(False)
cluster.detach(False)
cluster(str_cmd, waitForChild=True, stdoutflush=True, stderrflush=True)
if cluster.exitCode():
error.fatal(pipe_hypothermia, 'stageExec', cluster.stderr())
os.chdir(str_cwd)
return True
def run(self):
# we can pass a list of names
# default is all!
# start all plugins
print '-->run<--'
# use crun to run all jobs
shell = crun.crun()
shell.echo(False)
shell.echoStdOut(True)
shell.echoStdErr(True)
shell.detach(False)
shell.waitForChild(True)
for obj in self.plugins:
# feedname is the date
command = 'php ' + self.launcher
command += ' --username='******'name']
command += ' --password='******'password']
command += ' --feedname=' + d.datetime.now().strftime("%Y%m%d%H%M%S")
command += ' --feedid=-1'
command += ' --command="' + obj['executable'] + ' ' + obj['args'] + ' --output {OUTPUT}"'
print command
shell(command)
def f_stage3callback(**kwargs):
str_cwd = os.getcwd()
lst_subj = []
for key, val in kwargs.iteritems():
if key == 'subj': lst_subj = val
if key == 'obj': stage = val
for subj in lst_subj:
# find the relevant input files in each <subj> dir
os.chdir(subj)
l_ADC = misc.find(_str_adc)
if not l_ADC: error.fatal(pipe_hypothermia, 'noADC')
_str_ADCFile = l_ADC[0]
_str_outDir = 'outDir'
misc.mkdir(_str_outDir)
log = stage.log()
log('Scheduling masconorm for "%s: ADC"..\n' % (subj))
str_cmd = 'masconorm.py --input %s --mask %s/b0Brain_mask.nii.gz --outStem %s/adc' % (
_str_ADCFile,
_str_outDir,
_str_outDir)
cluster = crun.crun()
cluster.echo(False)
cluster.echoStdOut(False)
cluster.detach(False)
cluster(str_cmd, waitForChild=True, stdoutflush=True, stderrflush=True)
os.chdir(str_cwd)
return True
def f_stage0callback(**kwargs):
str_cwd = os.getcwd()
for key, val in kwargs.iteritems():
if key == 'operand': l_operand = val
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
count = 0
for i in range(1, len(l_operand)):
if i == 1: str_previousOutput = l_operand[i-1]
str_cumulativeOutput = "%d-%s" % (i, args.output)
str_cmd = 'mris_calc -o %s %s %s %s ; cp %s %s' % \
(str_cumulativeOutput, str_previousOutput, args.operation, l_operand[i],
str_cumulativeOutput, os.path.basename(str_previousOutput))
str_previousOutput = os.path.basename(str_previousOutput)
print str_cmd
cluster = crun.crun()
cluster.echo(False)
cluster.echoStdOut(False)
cluster.detach(False)
cluster(str_cmd, waitForChild=True, stdoutflush=True, stderrflush=True)
if cluster.exitCode():
error.fatal(pipe_mrisCalc, 'stageExec', cluster.stderr())
count += 1
if not count: error.fatal(pipe_mrisCalc, 'stageExec', "Insufficient operands found!")
return True
def f_stage2callback(**kwargs):
str_cwd = os.getcwd()
lst_subj = []
for key, val in kwargs.iteritems():
if key == 'subj': lst_subj = val
if key == 'obj': stage = val
for subj in lst_subj:
# find the relevant input files in each <subj> dir
os.chdir(subj)
l_B0 = misc.find(_str_b0)
l_ASL = misc.find(_str_asl)
if not l_B0: error.fatal(pipe_hypothermia, 'noB0')
if not l_ASL: error.fatal(pipe_hypothermia, 'noASL')
_str_B0File = l_B0[0]
_str_ASLFile = l_ASL[0]
_str_outDir = 'outDir'
_str_outFile = 'asl2b0.nii'
misc.mkdir(_str_outDir)
log = stage.log()
log('Scheduling masconorm for "%s: ASL"..\n' % (subj))
str_cmd = 'masconorm.py --input %s/asl2b0.nii.gz --mask %s/b0Brain_mask.nii.gz --outStem %s/asl' % (
_str_outDir,
_str_outDir,
_str_outDir)
#cluster = crun.crun_mosix()
cluster = crun.crun()
cluster.echo(False)
cluster.echoStdOut(False)
cluster.detach(False)
cluster(str_cmd, waitForChild=True, stdoutflush=True, stderrflush=True)
if cluster.exitCode():
error.fatal(pipe_hypothermia, 'stageExec', cluster.stderr())
os.chdir(str_cwd)
return True
def __init__(self, **kwargs):
'''
Basic constructor. Checks on named input args, checks that files
exist and creates directories.
'''
self._lw = 120
self._rw = 20
self._topDir = ''
self._str_remotePath = '~/chris'
self._str_remoteScript = 'headnode.bash'
self._str_remoteHost = 'mit.eofe4.edu'
self._str_remoteUser = '******'
self._str_remoteCrun = 'crun_hpc_mosix'
for key, value in kwargs.iteritems():
if key == 'remotePath': self._str_remotePath = value
if key == 'remoteScript': self._str_remoteScript = value
if key == 'remoteHost': self._str_remoteHost = value
if key == 'remoteUser': self._str_remoteUser = value
if key == 'remoteCrun': self._str_remoteCrun = value
# A local "shell"
self.OSshell = crun.crun()
self.OSshell.echo(False)
self.OSshell.echoStdOut(False)
self.OSshell.detach(False)
self.OSshell.waitForChild(True)
# The remote/scheduler shell
self.sshCluster = crun.crun(remoteUser=self._str_remoteUser,
remoteHost=self._str_remoteHost)
print("remote call stdout = %s" % self.sshCluster.stdout())
print("remote call stderr = %s" % self.sshCluster.stderr())
self.initialize()
def blockOnShellCmd(self,
astr_shellCmd,
astr_shellReturn,
astr_blockMsg,
astr_loopMsg,
atimeout=10):
'''
Block on a shell command.
This method will repeatedly poll a given <str_shellCmd> every
<atimeout> seconds until the command evaluates to <astr_shellReturn>.
The <astr_shellCmd> itself should not block, but simply evaluate a
condition and return a status.
It is typically used as either a pre- or post-condition filter.
ARGS
astr_shellCmd shell command to evaluate
astr_shellReturn the return from the shell command
indicating success
astr_blockMsg log message when block starts
astr_loopMsg log message while blocking
atimeout how long between checking astr_shellCmd
while blocking
RETURN
o True
'''
shell = crun.crun()
shell(astr_shellCmd)
blockLoop = 1
if shell.stdout().strip() != astr_shellReturn:
self._log(Colors.CYAN + astr_blockMsg + Colors.NO_COLOUR)
while 1:
time.sleep(atimeout)
shell(astr_shellCmd)
if shell.stdout().strip() == astr_shellReturn:
self._log('\n', syslog=False)
break
else:
str_loopMsg = Colors.BROWN + '(block duration = %ds; block processCount = %s) '% \
(blockLoop * atimeout, shell.stdout().strip()) + \
Colors.YELLOW + astr_loopMsg + Colors.NO_COLOUR
self._log(str_loopMsg)
loopMsgLen = len(str_loopMsg)
syslogLen = len(self._log.str_syslog())
for i in range(0, loopMsgLen + syslogLen):
self._log('\b', syslog=False)
blockLoop += 1
return True
def blockOnShellCmd(self, astr_shellCmd, astr_shellReturn,
astr_blockMsg,
astr_loopMsg,
atimeout = 10):
'''
Block on a shell command.
This method will repeatedly poll a given <str_shellCmd> every
<atimeout> seconds until the command evaluates to <astr_shellReturn>.
The <astr_shellCmd> itself should not block, but simply evaluate a
condition and return a status.
It is typically used as either a pre- or post-condition filter.
ARGS
astr_shellCmd shell command to evaluate
astr_shellReturn the return from the shell command
indicating success
astr_blockMsg log message when block starts
astr_loopMsg log message while blocking
atimeout how long between checking astr_shellCmd
while blocking
RETURN
o True
'''
shell = crun.crun()
shell(astr_shellCmd)
blockLoop = 1
if shell.stdout().strip() != astr_shellReturn:
self._log(Colors.CYAN + astr_blockMsg + Colors.NO_COLOUR)
while 1:
time.sleep(atimeout)
shell(astr_shellCmd)
if shell.stdout().strip() == astr_shellReturn:
self._log('\n', syslog=False)
break
else:
str_loopMsg = Colors.BROWN + '(block duration = %ds; block processCount = %s) '% \
(blockLoop * atimeout, shell.stdout().strip()) + \
Colors.YELLOW + astr_loopMsg + Colors.NO_COLOUR
self._log(str_loopMsg)
loopMsgLen = len(str_loopMsg)
syslogLen = len(self._log.str_syslog())
for i in range(0, loopMsgLen+syslogLen): self._log('\b', syslog=False)
blockLoop += 1
return True
def __init__(self, **kwargs):
'''
Sub-class constuctor. Currently sets an internal sub-class
_shell object, and then calls the super-class constructor.
'''
self._shell = crun.crun()
Stage.__init__(self, **kwargs)
# The following flags force flushing of stdout/stderr streams
# while executing shell command. These are useful for "slow"
# shell operations that take a long time to return but still
# generate output to stdout/stderr.
self._b_stdoutflush = True
self._b_stderrflush = True
for key, value in kwargs.iteritems():
if key == 'stdoutflush': self._b_stdoutflush = value
if key == 'stderrflush': self._b_stderrflush = value
def f_stage0callback(**kwargs):
lst_subj = []
for key, val in kwargs.iteritems():
if key == 'subj': lst_subj = val
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
lst_hemi = pipeline.l_hemisphere()
lst_surface = pipeline.l_surface()
lst_curv = pipeline.l_curv()
if int(args.partitions) > _maxPartitions:
error.fatal(hbwm, 'Partition')
for pipeline._str_subj in lst_subj:
for pipeline._str_hemi in lst_hemi:
for pipeline._str_surface in lst_surface:
for pipeline._str_curv in lst_curv:
log = stage.log()
log('Processing %s: %s.%s, %s...\n' % \
(pipeline.subj(), pipeline.hemi(), pipeline.surface(), pipeline.curv()))
str_hostOnlySpec = ''
if len(args.host):
str_hostOnlySpec = "--host %s " % args.host
log('Locking jobs to only run on host -->%s<--\n' %
args.host)
str_debug = ""
if args.b_debug: str_debug = " --debug "
str_queue = ""
if args.queue: str_queue = " --queue %s " % args.queue
str_cmd = "~/src/scripts/hbwm.py -v 10 -s %s %s -r -m %s -f %s -c %s -p %s --cluster %s %s %s %s" % \
(args.stages, str_hostOnlySpec,
pipeline.hemi(), pipeline.surface(), pipeline.curv(), args.partitions,
args.cluster, str_debug, str_queue, pipeline.subj())
print str_cmd
shell = crun.crun()
shell.echo(False)
shell.echoStdOut(False)
shell.detach(False)
shell(str_cmd,
waitForChild=True,
stdoutflush=True,
stderrflush=True)
if shell.exitCode():
error.fatal(hbwm, 'stageExec', shell.stderr())
os.chdir(pipeline.startDir())
return True
def __init__(self, **kwargs):
'''
Sub-class constuctor. Currently sets an internal sub-class
_shell object, and then calls the super-class constructor.
'''
self._shell = crun.crun()
Stage.__init__(self, **kwargs)
# The following flags force flushing of stdout/stderr streams
# while executing shell command. These are useful for "slow"
# shell operations that take a long time to return but still
# generate output to stdout/stderr.
self._b_stdoutflush = True
self._b_stderrflush = True
for key, value in kwargs.iteritems():
if key == 'stdoutflush': self._b_stdoutflush = value
if key == 'stderrflush': self._b_stderrflush = value
def f_stage0callback(**kwargs):
lst_subj = []
for key, val in kwargs.iteritems():
if key == 'subj': lst_subj = val
if key == 'obj': stage = val
if key == 'pipe': pipeline = val
lst_hemi = pipeline.l_hemisphere()
lst_surface = pipeline.l_surface()
lst_curv = pipeline.l_curv()
if int(args.partitions) > _maxPartitions:
error.fatal(hbwm, 'Partition')
for pipeline._str_subj in lst_subj:
for pipeline._str_hemi in lst_hemi:
for pipeline._str_surface in lst_surface:
for pipeline._str_curv in lst_curv:
log = stage.log()
log('Processing %s: %s.%s, %s...\n' % \
(pipeline.subj(), pipeline.hemi(), pipeline.surface(), pipeline.curv()))
str_hostOnlySpec = ''
if len(args.host):
str_hostOnlySpec = "--host %s " % args.host
log('Locking jobs to only run on host -->%s<--\n' % args.host)
str_debug = ""
if args.b_debug: str_debug = " --debug "
str_queue = ""
if args.queue: str_queue = " --queue %s " % args.queue
str_cmd = "~/src/scripts/hbwm.py -v 10 -s %s %s -r -m %s -f %s -c %s -p %s --cluster %s %s %s %s" % \
(args.stages, str_hostOnlySpec,
pipeline.hemi(), pipeline.surface(), pipeline.curv(), args.partitions,
args.cluster, str_debug, str_queue, pipeline.subj())
print str_cmd
shell = crun.crun()
shell.echo(False)
shell.echoStdOut(False)
shell.detach(False)
shell(str_cmd, waitForChild=True, stdoutflush=True, stderrflush=True)
if shell.exitCode():
error.fatal(hbwm, 'stageExec', shell.stderr())
os.chdir(pipeline.startDir())
return True
def f_stage4callback(**kwargs):
str_cwd = os.getcwd()
lst_subj = []
for key, val in kwargs.iteritems():
if key == 'subj': lst_subj = val
if key == 'obj': stage = val
for subj in lst_subj:
# find the relevant input files in each <subj> dir
_str_outDir = 'outDir'
misc.mkdir(_str_outDir)
os.chdir("%s/outDir" % subj)
log = stage.log()
log('Scheduling MatLAB basac analysis for subject "%s"...\n' % (subj))
str_cmd = 'eval "/chb/pices/arch/x86_64-Linux/bin/matlab -nodesktop -nosplash -nojvm -r \\\"c = basac_drive(\'%s\'); exit\\\"" |tee basac_process.log' % (
os.getcwd()
)
cluster = crun.crun()
cluster.echo(False)
cluster.echoStdOut(False)
cluster.detach(False)
cluster(str_cmd, waitForChild=True, stdoutflush=True, stderrflush=True)
os.chdir(str_cwd)
return True
help='comma separated curvature function list to process')
parser.add_argument('--statFunc', '-f',
dest='statFunc',
action='store',
default='ptile-raw',
help='comma separated statistical function list to process')
parser.add_argument('--hemi', '-m',
dest='hemi',
action='store',
default='lh,rh',
help='comma separated hemisphere list to process')
args = parser.parse_args()
# A generic "shell"
OSshell = crun.crun()
OSshell.echo(False)
OSshell.echoStdOut(False)
OSshell.detach(False)
OSshell.waitForChild(True)
roibsanalyze = FNNDSC_ROIBSANALYZE(
rsampleList = args.l_resample,
annot = args.annotation,
outDir = args.outDir,
stages = args.stages,
pval = args.pval,
group = args.group,
surface = args.surface,
statFunc = args.statFunc,
curvFunc = args.curvFunc,
help='number of instances of <cmd> to schedule')
parser.add_argument('--debug', '-d',
dest='b_debug',
action="store_true",
default=False,
help='turn on debugging info')
parser.add_argument('--cluster', '-l',
dest='cluster',
action='store',
default='PICES',
help='destination cluster to schedule jobs on')
args = parser.parse_args()
# A "localhost" shell that this script can use to run shell commands on.
OSshell = crun.crun()
OSshell.echo(False)
OSshell.echoStdOut(False)
OSshell.detach(False)
hpc = CLUSTER(
jobs = args.jobs,
logTo = 'hpctest.log',
syslog = True,
logTee = True
)
hpc.verbosity(args.verbosity)
pipeline = hpc.pipeline()
pipeline.poststdout(True)
def execCmd(self, cmdList=None, userErrStr='Plugin returned error!'):
"""
This method executes a list of commands (eg. mri_convert) either by scheduling them on a cluster
or just running them locally depending on Chris configuration
"""
if (cmdList is None):
raise ValueError("A list of cmd strings must be passed on")
cmdIds = []
exitCodeFilePaths = [];
chrisRunDir = self.chrisRunDir
envFile = open(chrisRunDir + '/chris.env')
envStr = envFile.read()
envFile.close()
clType = self.clusterType
for cmd, outFileName in itertools.izip(cmdList, self.outputFileNames):
#create stderr, stdout, and exit code log for each cmd execution
ix = outFileName.rfind('.')
cmdId = outFileName[:ix]
cmdIds.append(cmdId)
out = '%s/%s' % (chrisRunDir, cmdId)
exitCodeFilePaths.append(out + '-exitCode.log')
if clType == 'local':
cmd = '%s 2>%s.err >%s.std || (echo $? > %s-exitCode.log)' % (cmd, out, out, out)
else:
#write a dummy cmdId.log file too (this is used later to block the python process until all jobs have finished)
cmd = '%s 2>%s.err >%s.std || (echo $? > %s-exitCode.log);touch %s.log' % (cmd, out, out, out, out)
misc.file_writeOnce(out + '.run', envStr + '\n' + cmd)
os.system('chmod 755 ' + out + '.run')
if clType == 'local':
#jobs are not scheduled on a cluster but run locally
shell = crun.crun()
shell.echo(False)
shell.waitForChild(True)
for cmdId in cmdIds:
shell("/bin/bash " + chrisRunDir + '/' + cmdId + '.run')
else:
#jobs are scheduled on a cluster
remUser = self.remoteUser
remHost = self.remoteHost
remUserId = self.remoteUserIdentity
shell = eval('crun.' + clType.lower() + '(remoteUser=remUser, remoteHost=remHost, remoteUserIdentity=remUserId)')
for cmdId in cmdIds:
shell("/bin/bash " + chrisRunDir + '/' + cmdId + '.run', stdoutflush=True, stderrflush=True)
time.sleep(0.5)
#execute while loop until all cmds have written a file with a name cmdId.log in the chris run dir
allHaveWritten = False
while not allHaveWritten:
dirList = os.listdir(chrisRunDir)
#list of cmd ids found in the chris run dir
foundCmdIds = []
for cmdId in cmdIds:
for name in dirList:
if name == cmdId + '.log':
foundCmdIds.append(cmdId)
#remove found dummy log file
os.remove(chrisRunDir + '/' + name)
break;
if len(foundCmdIds) == len(cmdIds):
allHaveWritten = True
else:
for cmdId in foundCmdIds:
#remove already found ids
cmdIds.remove(cmdId)
time.sleep(10)
for path, outFileName in itertools.izip(exitCodeFilePaths, self.outputFileNames):
# check for existence of chrisRunDir/cmdId-exitCode.log
if os.path.isfile(path):
rfile = open(path)
if rfile.read(1) == "0":
shutil.copyfile(os.path.join(self.tempdir, outFileName), os.path.join(self.options.output, outFileName))
else:
ix = outFileName.rfind('.')
misc.file_writeOnce(os.path.join(chrisRunDir, 'ERROR-user-' + outFileName[:ix] + '.err'), userErrStr + ' ' + outFileName)
rfile.close()
else:
shutil.copyfile(os.path.join(self.tempdir, outFileName), os.path.join(self.options.output, outFileName))
def __init__(self, **kwargs):
'''
Basic constructor. Checks on named input args, checks that files
exist and creates directories.
'''
self._lw = 120
self._rw = 20
self._topDir = ''
self._l_cmd = []
self._b_sleepCmd = False
self._sleepMaxLength = 0
self._numberOfChildren = 1
self._str_remotePath = '~/chris/src/cruntesting'
self._str_remoteCrun = 'crun_hpc_slurm'
self._str_queue = ''
self._str_schedulerStdOutDir = 'jobOutDir'
self._str_schedulerStdErrDir = 'jobErrDir'
self._b_cleanup = True
self._b_internalWait = True
self._pythonpath = ''
# A local "shell"
self.OSshell = crun.crun()
self.OSshell.echo(False)
self.OSshell.echoStdOut(False)
self.OSshell.detach(False)
self.OSshell.waitForChild(True)
self.OSshell('whoami')
self._str_remoteUser = self.OSshell.stdout()
self.OSshell('hostname -s')
self._str_remoteHost = self.OSshell.stdout()
for key, value in kwargs.iteritems():
if key == 'crun': self._str_remoteCrun = value
if key == 'queue': self._str_queue = value
if key == 'headnode': self._str_remoteHost = value
if key == 'user': self._str_remoteUser = value
if key == 'cnodescratchpath': self._str_cnodescratchpath = value
if key == 'jobOutDir': self._str_schedulerStdOutDir = value
if key == 'jobErrDir': self._str_schedulerStdErrDir = value
if key == 'cmd': self._l_cmd = value
if key == 'children': self._numberOfChildren = int(value)
if key == 'sleepMaxLength': self._sleepMaxLength = int(value)
if key == 'b_cleanup': self._b_cleanup = value
if key == 'pythonpath': sys.path.append(value)
if key == 'b_internalWait': self._b_internalWait = int(value)
# The remote/scheduler shell
print('remoteUser = %s' % self._str_remoteUser)
print('remoteHost = %s' % self._str_remoteHost)
print('schedulerStdOutDir = %s' % self._str_schedulerStdOutDir)
print('schedulerStdErrDir = %s' % self._str_schedulerStdErrDir)
print('crun.' + self._str_remoteCrun + '(remoteUser=self._str_remoteUser,remoteHost=self._str_remoteHost,schedulerStdOutDir=self._str_schedulerStdOutDir,schedulerStdErrDir=self._str_schedulerStdErrDir)')
print("in computenode.py stdoutDir = %s stderrDir =%s" % (self._str_schedulerStdOutDir, self._str_schedulerStdErrDir))
self.sshCluster = eval('crun.' + self._str_remoteCrun + '(remoteUser=self._str_remoteUser,remoteHost=self._str_remoteHost,schedulerStdOutDir=self._str_schedulerStdOutDir,schedulerStdErrDir=self._str_schedulerStdErrDir)')
if len(self._str_queue):
self.sshCluster.queueName(self._str_queue)
self.initialize()
