def test_pbs_remove(self):
'''
test Pbs remove
remove has no return value so we can't really assert anything
interesting.
'''
with patch('pbs.pbs_default'):
with patch('pbs.pbs_connect'):
with patch('pbs.pbs_deljob', return_value=True):
o = hrr.Pbs(None)
o.remove()
with patch('pbs.pbs_deljob', return_value=False):
o = hrr.Pbs(None)
o.remove()
def test_pbs_info(self):
'''test Pbs info -- though it doesn't do it yet.'''
statjob = [
Attribs(name='123.master.domain',
attribs=[
Attrib('Job_Name', 'HOD_job'),
Attrib('job_state', 'R'),
Attrib('exec_host', 'some-hosts'),
])
]
expected = [{
'id': '123.master.domain',
'Job_Name': 'HOD_job',
'job_state': 'R',
'exec_host': 'some-hosts'
}]
with patch('pbs.pbs_default'):
with patch('pbs.pbs_connect'):
with patch('pbs.pbs_statjob', return_value=statjob):
o = hrr.Pbs(None)
info = o.info('123')
self.assertEqual(info[0]['id'], expected[0]['id'])
self.assertEqual(info[0]['Job_Name'], expected[0]['Job_Name'])
self.assertEqual(info[0]['job_state'], expected[0]['job_state'])
self.assertEqual(info[0]['exec_host'], expected[0]['exec_host'])
self.assertEqual(info, expected)
def test_pbs_state_none(self):
statjob = []
with patch('pbs.pbs_default'):
with patch('pbs.pbs_connect'):
with patch('pbs.pbs_statjob', return_value=statjob):
o = hrr.Pbs(None)
state = o.state()
self.assertEqual(len(state), 0)
def test_pbs_get_ppn_none(self):
'''test Pbs get_ppn'''
o = hrr.Pbs(None)
nodes = {}
with patch('PBSQuery.PBSQuery',
return_value=Mock(getnodes=lambda: nodes)):
self.assertTrue(o.get_ppn() is None)
def run(self, args):
"""Run 'connect' subcommand."""
optparser = ConnectOptions(go_args=args,
envvar_prefix=self.envvar_prefix,
usage=self.usage_txt)
try:
if len(optparser.args) > 1:
label = optparser.args[1]
else:
self.report_error("No label provided.")
print "Connecting to HOD cluster with label '%s'..." % label
try:
jobid = cluster_jobid(label)
env_script = cluster_env_file(label)
except ValueError as err:
self.report_error(err)
print "Job ID found: %s" % jobid
pbs = rm_pbs.Pbs(optparser)
jobs = pbs.state()
pbsjobs = [job for job in jobs if job.jobid == jobid]
if len(pbsjobs) == 0:
self.report_error("Job with job ID '%s' not found by pbs.",
jobid)
elif len(pbsjobs) > 1:
self.report_error("Multiple jobs found with job ID '%s': %s",
jobid, pbsjobs)
pbsjob = pbsjobs[0]
if pbsjob.state == ['Q', 'H']:
# This should never happen since the hod.d/<jobid>/env file is
# written on cluster startup. Maybe someone hacked the dirs.
self.report_error(
"Cannot connect to cluster with job ID '%s' yet. It is still queued.",
jobid)
else:
print "HOD cluster '%s' @ job ID %s appears to be running..." % (
label, jobid)
print "Setting up SSH connection to %s..." % pbsjob.hosts
# -i: interactive non-login shell
cmd = [
'ssh', '-t', pbsjob.hosts, 'exec', 'bash', '--rcfile',
env_script, '-i'
]
self.log.info("Logging in using command: %s", ' '.join(cmd))
os.execvp('/usr/bin/ssh', cmd)
except StandardError as err:
self._log_and_raise(err)
return 0
def test_pbs_submit(self):
'''test Pbs submit'''
with patch('pbs.pbs_default'):
with patch('pbs.pbs_connect'):
with patch('pbs.pbs_submit', return_value=True):
o = hrr.Pbs(dict(name='test-job-name'))
o.get_ppn = lambda: 24
o.header()
o.submit('hostname')
def run(self, args):
"""Run 'clean' subcommand."""
optparser = CleanOptions(go_args=args, envvar_prefix=self.envvar_prefix, usage=self.usage_txt)
try:
pbs = rm_pbs.Pbs(optparser)
state = pbs.state()
labels = hc.known_cluster_labels()
rm_master = rm_pbs.master_hostname()
info = hc.mk_cluster_info_dict(labels, state, master=rm_master)
hc.clean_cluster_info(rm_master, info)
except StandardError as err:
self._log_and_raise(err)
return 0
def test_pbs_match_filter(self):
'''test Pbs match_filter'''
job = {
'id': '123.master.domain',
'Job_Name': 'HOD_job',
'job_state': 'R',
'exec_host': 'some-hosts'
}
with patch('pbs.pbs_default'):
with patch('pbs.pbs_connect'):
o = hrr.Pbs(None)
self.assertTrue(o.match_filter(job, {'Job_Name': '^HOD'}))
self.assertFalse(o.match_filter(job, {'Job_Name': 'HOD$'}))
def test_pbs_state_one(self):
statjob = [
Attribs(name='123.master.domain',
attribs=[
Attrib('job_state', 'R'),
Attrib('exec_host', 'node1.domain/1+node1.domain/2'),
])
]
with patch('pbs.pbs_default'):
with patch('pbs.pbs_connect'):
with patch('pbs.pbs_statjob', return_value=statjob):
o = hrr.Pbs(None)
state = o.state()
self.assertEqual(len(state), 1)
self.assertEqual(state[0].jobid, '123.master.domain')
self.assertEqual(state[0].state, 'R')
self.assertEqual(state[0].hosts, 'node1.domain')
def test_pbs_get_ppn(self):
'''test Pbs get_ppn'''
o = hrr.Pbs(None)
nodes = {
'node1': {
'np': ['24'],
'state': ['free']
},
'node2': {
'np': ['24'],
'state': ['job-exclusive']
}
}
with patch('PBSQuery.PBSQuery',
return_value=Mock(getnodes=lambda: nodes)):
self.assertEqual(24, o.get_ppn())
def run(self, args):
"""Run 'destroy' subcommand."""
optparser = DestroyOptions(go_args=args,
envvar_prefix=self.envvar_prefix,
usage=self.usage_txt)
try:
label, jobid = None, None
if len(optparser.args) > 1:
label = optparser.args[1]
print "Destroying HOD cluster with label '%s'..." % label
else:
self.report_error("No label provided.")
try:
jobid = cluster_jobid(label)
print "Job ID: %s" % jobid
except ValueError as err:
self.report_error(err)
# try to figure out job state
job_state = None
pbs = rm_pbs.Pbs(optparser)
jobs = pbs.state()
pbsjobs = [job for job in jobs if job.jobid == jobid]
self.log.debug("Matching jobs for job ID '%s': %s", jobid, pbsjobs)
if len(pbsjobs) == 1:
job_state = pbsjobs[0].state
print "Job status: %s" % job_state
elif len(pbsjobs) == 0:
print "(job no longer found)"
else:
self.report_error("Multiple jobs found with job ID '%s': %s",
jobid, pbsjobs)
# request confirmation is case the job is currently running
if job_state == 'R':
resp = raw_input(
"Confirm destroying the *running* HOD cluster with label '%s'? [y/n]: "
% label)
if resp != 'y':
print "(destruction aborted)"
return
elif job_state in ['C', 'E']:
print "(job has already ended/completed)"
job_state = None
print "\nStarting actual destruction of HOD cluster with label '%s'...\n" % label
# actually destroy HOD cluster by deleting job and removing cluster info dir and local work dir
if job_state is not None:
# if job was not successfully deleted, pbs.remove will print an error message
if pbs.remove(jobid):
print "Job with ID %s deleted." % jobid
rm_cluster_localworkdir(label)
if cluster_info_exists(label):
rm_cluster_info(label)
print "\nHOD cluster with label '%s' (job ID: %s) destroyed." % (
label, jobid)
except StandardError as err:
self._log_and_raise(err)
return 0
def test_pbs_init(self):
'''test Pbs init function'''
o = hrr.Pbs(None)
def test_pbs_header(self):
'''test Pbs header'''
o = hrr.Pbs(None)
hdr = o.header()