def test_monitor_job_running(self, retry, get_connection):
conn = get_connection.return_value.__enter__.return_value
job_id = 'dummy'
cluster = {
'_id': 'jill',
'type': 'ec2',
'name': 'dummy',
'config': {
'_id': 'dummy',
'scheduler': {
'type': 'sge'
}
}
}
job_model = {
'_id': job_id,
'queueJobId': '1',
'name': 'dummy',
'output': []
}
conn.execute.return_value = [ 'job-ID prior name user state submit/start at queue slots ja-task-ID',
'-----------------------------------------------------------------------------------------',
'1 0.00000 hostname sgeadmin r 09/09/2009 14:58:14 1']
def _get_status(url, request):
content = {
'status': 'running'
}
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
self._get_status_called = True
return httmock.response(200, content, headers, request=request)
def _set_status(url, request):
expected = {'status': 'running', 'timings': {}, 'output': []}
self._set_status_called = json.loads(request.body) == expected
if not self._set_status_called:
six.print_(json.loads(request.body), file=sys.stderr)
return httmock.response(200, None, {}, request=request)
status_url = '/api/v1/jobs/%s/status' % job_id
get_status = httmock.urlmatch(
path=r'^%s$' % status_url, method='GET')(_get_status)
status_update_url = '/api/v1/jobs/%s' % job_id
set_status = httmock.urlmatch(
path=r'^%s$' % status_update_url, method='PATCH')(_set_status)
with httmock.HTTMock(get_status, set_status):
job.monitor_job(cluster, job_model, **{'girder_token': 's', 'log_write_url': 1})
self.assertTrue(self._get_status_called, 'Expect get status endpoint to be hit')
self.assertTrue(self._set_status_called, 'Expect set status endpoint to be hit')
def mock_response(
content,
url=None,
path="",
headers=None,
response_url=None,
status_code=200,
cookies=None,
func=None,
):
"""Universal handler for specify mocks inplace."""
if func is None:
def mocked(url, request):
mock = response(status_code=status_code,
content=content,
request=request,
headers=headers)
if cookies:
mock.cookies = cookies
mock.url = response_url if response_url else url
return mock
else:
mocked = func
if url:
parsed = urlparse(url)
return urlmatch(netloc=parsed.netloc, path=parsed.path)(func=mocked)
elif path:
return urlmatch(path=path)(func=mocked)
else:
return all_requests(func=mocked)
def test_monitor_job_terminated(self, get_connection):
conn = get_connection.return_value.__enter__.return_value
job_id = 'dummy'
cluster = {
'_id': 'bob',
'type': 'ec2',
'name': 'dummy',
'config': {
'_id': 'dummy',
'scheduler': {
'type': 'sge'
}
}
}
job_model = {
'_id': job_id,
'queueJobId': 'dummy',
'name': 'dummy',
'output': []
}
conn.execute.return_value = 'qstat output'
def _get_status(url, request):
content = {
'status': 'terminating'
}
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
self._get_status_called = True
return httmock.response(200, content, headers, request=request)
def _set_status(url, request):
expected = {u'status': u'terminated', u'timings': {}, u'output': []}
self._set_status_called = json.loads(request.body) == expected
return httmock.response(200, None, {}, request=request)
status_url = '/api/v1/jobs/%s/status' % job_id
get_status = httmock.urlmatch(
path=r'^%s$' % status_url, method='GET')(_get_status)
status_update_url = '/api/v1/jobs/%s' % job_id
set_status = httmock.urlmatch(
path=r'^%s$' % status_update_url, method='PATCH')(_set_status)
with httmock.HTTMock(get_status, set_status):
job.monitor_job(cluster, job_model, **{'girder_token': 's', 'log_write_url': 1})
self.assertTrue(self._get_status_called, 'Expect get status endpoint to be hit')
self.assertTrue(self._set_status_called, 'Expect set status endpoint to be hit')
def test_key_generate(self):
cluster = {
'_id': '55c3a698f6571011a48f6817'
}
key_path = os.path.join(cumulus.config.ssh.keyStore, cluster['_id'])
try:
os.remove(key_path)
except OSError:
pass
def _update(url, request):
request_body = json.loads(request.body)
passphrase = parse('config.ssh.passphrase').find(request_body)
public_key = parse('config.ssh.publicKey').find(request_body)
status = request_body['status'] == 'created'
self._update = passphrase and public_key and status
return httmock.response(200, None, {}, request=request)
update_url = '/api/v1/clusters/%s' % cluster['_id']
update = httmock.urlmatch(
path=r'^%s$' % update_url, method='PATCH')(_update)
with httmock.HTTMock(update):
key.generate_key_pair(cluster)
self.assertTrue(os.path.exists(key_path), 'Key was not created')
os.remove(key_path)
self.assertTrue(self._update, 'Update was not called')
def setUp(self):
status_url = '%s/login' % newt_base_url
data = {
'username': NewtClusterConnectionTestCase.USER,
'password': NewtClusterConnectionTestCase.PASSWORD
}
r = requests.post(status_url, data=data)
json_resp = r.json()
self.assertTrue(json_resp['auth'])
self.session_id = json_resp['newt_sessionid']
self._cluster = {'type': 'newt', 'config': {'host': 'cori'}}
self._girder_token = 'dummy'
def session_id(url, request):
return self._session_id(url, request)
url = '/api/v1/newt/sessionId'
self.me = httmock.urlmatch(path=r'^%s$' % url,
method='GET')(session_id)
self.scratch_dir = '/global/cscratch1/sd/%s' % NewtClusterConnectionTestCase.USER
self.test_data = 'nothing to see here!'
self.test_case_dir = '%s/cumulus' % self.scratch_dir
self.test_file_path = '%s/test.txt' % self.test_case_dir
self.test_dir = '%s/cumulus' % self.test_case_dir
# Create directory for test case
with httmock.HTTMock(self.me):
with get_connection(self._girder_token, self._cluster) as conn:
conn.mkdir(self.test_case_dir)
def testPatch(self):
patchUrl = 'patch'
patchRequest = {
'valid': False
}
# Test json request
jsonBody = {
'foo': 'bar'
}
def _patchJson(url, request):
body = request.body
if isinstance(body, six.binary_type):
body = body.decode('utf8')
patchRequest['valid'] = json.loads(body) == jsonBody
return httmock.response(200, {}, {}, request=request)
patch = httmock.urlmatch(
path=r'^.*%s$' % patchUrl, method='PUT')(_patchJson)
with httmock.HTTMock(patch):
client = girder_client.GirderClient()
client.put(patchUrl, json=jsonBody)
# Check we got the right request
self.assertTrue(patchRequest['valid'])
# Now try raw message body
patchRequest['valid'] = False
rawBody = 'raw'
def _patchRaw(url, request):
patchRequest['valid'] = request.body == rawBody
return httmock.response(200, {}, {}, request=request)
patch = httmock.urlmatch(
path=r'^.*%s$' % patchUrl, method='PUT')(_patchRaw)
with httmock.HTTMock(patch):
client = girder_client.GirderClient()
client.put(patchUrl, data=rawBody)
# Check we got the right request
self.assertTrue(patchRequest['valid'])
def testPatch(self):
patchUrl = 'patch'
patchRequest = {
'valid': False
}
# Test json request
jsonBody = {
'foo': 'bar'
}
def _patchJson(url, request):
body = request.body
if isinstance(body, bytes):
body = body.decode('utf8')
patchRequest['valid'] = json.loads(body) == jsonBody
return httmock.response(200, {}, {}, request=request)
patch = httmock.urlmatch(
path=r'^.*%s$' % patchUrl, method='PUT')(_patchJson)
with httmock.HTTMock(patch):
client = girder_client.GirderClient()
client.put(patchUrl, json=jsonBody)
# Check we got the right request
self.assertTrue(patchRequest['valid'])
# Now try raw message body
patchRequest['valid'] = False
rawBody = 'raw'
def _patchRaw(url, request):
patchRequest['valid'] = request.body == rawBody
return httmock.response(200, {}, {}, request=request)
patch = httmock.urlmatch(
path=r'^.*%s$' % patchUrl, method='PUT')(_patchRaw)
with httmock.HTTMock(patch):
client = girder_client.GirderClient()
client.put(patchUrl, data=rawBody)
# Check we got the right request
self.assertTrue(patchRequest['valid'])
def json_fixture(path_match=None, response_data={}, status_code=200):
def fixture_response(url, req):
return {'status_code': status_code,
'content': response_data,
'headers': {'content-type': 'application/json'}}
if path_match is None:
return all_requests(fixture_response)
return urlmatch(path=path_match)(fixture_response)
def json_fixture(path_match=None, response_data={}, status_code=200):
def fixture_response(url, req):
return {
'status_code': status_code,
'content': response_data,
'headers': {
'content-type': 'application/json'
}
}
if path_match is None:
return all_requests(fixture_response)
return urlmatch(path=path_match)(fixture_response)
def test_key_generate(self, get_ec2_client):
ec2_client = get_ec2_client.return_value
ec2_client.create_key_pair.side_effect = Exception('some error')
profile = {
'_id': '55c3a698f6571011a48f6817',
'userId': '55c3a698f6571011a48f6818',
'name': 'profile'
}
key_path = os.path.join(cumulus.config.ssh.keyStore, profile['_id'])
try:
os.remove(key_path)
except OSError:
pass
def _update(url, request):
request_body = json.loads(request.body.decode('utf8'))
status = parse('status').find(request_body)
if status:
status = status[0].value
self._update = status == self._expected_status
if status == 'error':
self._errorMessage = request_body['errorMessage']
return httmock.response(200, None, {}, request=request)
update_url = '/api/v1/user/%s/aws/profiles/%s' % (profile['userId'],
profile['_id'])
update = httmock.urlmatch(
path=r'^%s$' % update_url, method='PATCH')(_update)
self._expected_status = 'error'
with httmock.HTTMock(update):
key.generate_key_pair(profile, 'girder-token')
self.assertTrue(self._update, 'Update was not called')
self.assertTrue(self._errorMessage, 'No errorMessage set')
# Now mock out EC2 and check for success
self._update = False
self._expected_status = 'available'
ec2_client.create_key_pair.side_effect = None
ec2_client.create_key_pair.return_value = {
'KeyMaterial': 'dummy'
}
with httmock.HTTMock(update):
key.generate_key_pair(profile, 'girder-token')
def test_key_generate(self, get_ec2_client):
ec2_client = get_ec2_client.return_value
ec2_client.create_key_pair.side_effect = Exception('some error')
profile = {
'_id': '55c3a698f6571011a48f6817',
'userId': '55c3a698f6571011a48f6818',
'name': 'profile'
}
key_path = os.path.join(cumulus.config.ssh.keyStore, profile['_id'])
try:
os.remove(key_path)
except OSError:
pass
def _update(url, request):
request_body = json.loads(request.body)
status = parse('status').find(request_body)
if status:
status = status[0].value
self._update = status == self._expected_status
if status == 'error':
self._errorMessage = request_body['errorMessage']
return httmock.response(200, None, {}, request=request)
update_url = '/api/v1/user/%s/aws/profiles/%s' % (profile['userId'],
profile['_id'])
update = httmock.urlmatch(
path=r'^%s$' % update_url, method='PATCH')(_update)
self._expected_status = 'error'
with httmock.HTTMock(update):
key.generate_key_pair(profile, 'girder-token')
self.assertTrue(self._update, 'Update was not called')
self.assertTrue(self._errorMessage, 'No errorMessage set')
# Now mock out EC2 and check for success
self._update = False
self._expected_status = 'available'
ec2_client.create_key_pair.side_effect = None
ec2_client.create_key_pair.return_value = {
'KeyMaterial': 'dummy'
}
with httmock.HTTMock(update):
key.generate_key_pair(profile, 'girder-token')
def setUp(self):
status_url = '%s/login' % newt_base_url
data = {
'username': NewtClusterConnectionTestCase.USER,
'password': NewtClusterConnectionTestCase.PASSWORD
}
r = requests.post(status_url, data=data)
json_resp = r.json()
self.assertTrue(json_resp['auth'])
self.session_id = json_resp['newt_sessionid']
self._cluster = {
'type': 'newt',
'config': {
'host': 'cori'
}
}
self._girder_token = 'dummy'
def session_id(url, request):
return self._session_id(url, request)
url = '/api/v1/newt/sessionId'
self.me = httmock.urlmatch(
path=r'^%s$' % url, method='GET')(session_id)
self.scratch_dir = '/global/cscratch1/sd/%s' % NewtClusterConnectionTestCase.USER
self.test_data = 'nothing to see here!'
self.test_case_dir = '%s/cumulus' % self.scratch_dir
self.test_file_path = '%s/test.txt' % self.test_case_dir
self.test_dir = '%s/cumulus' % self.test_case_dir
# Create directory for test case
with httmock.HTTMock(self.me):
with get_connection(self._girder_token, self._cluster) as conn:
conn.mkdir(self.test_case_dir)
def test_monitor_jobs_queued_no_retry(self, retry, get_connection):
conn = get_connection.return_value.__enter__.return_value
cluster = {
'_id': 'lost',
'type': 'ec2',
'name': 'dummy',
'config': {
'_id': 'dummy',
'scheduler': {
'type': 'sge'
}
}
}
job1_id = 'dummy1'
job1_model = {
'_id': job1_id,
'queueJobId': '1',
'name': 'dummy',
'output': []
}
job2_id = 'dummy1'
job2_model = {
'_id': job2_id,
'queueJobId': '2',
'name': 'dummy',
'output': []
}
conn.execute.return_value = [ 'job-ID prior name user state submit/start at queue slots ja-task-ID',
'-----------------------------------------------------------------------------------------']
self._get_status_calls = {}
self._set_status_calls = {}
def _get_status(url, request):
content = {
'status': 'complete'
}
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
job_id = url.path.split('/')[-2]
self._get_status_calls[job_id] = True
return httmock.response(200, content, headers, request=request)
def _set_status(url, request):
expected = {'status': 'complete', 'timings': {}, 'output': []}
job_id = url.path.split('/')[-1]
self._set_status_calls[job_id] = True
self._set_status_called = json.loads(request.body) == expected
return httmock.response(200, None, {}, request=request)
job1_status_url = '/api/v1/jobs/%s/status' % job1_id
job1_get_status = httmock.urlmatch(
path=r'^%s$' % job1_status_url, method='GET')(_get_status)
job1_status_update_url = '/api/v1/jobs/%s' % job1_id
job1_set_status = httmock.urlmatch(
path=r'^%s$' % job1_status_update_url, method='PATCH')(_set_status)
job2_status_url = '/api/v1/jobs/%s/status' % job2_id
job2_get_status = httmock.urlmatch(
path=r'^%s$' % job2_status_url, method='GET')(_get_status)
job2_status_update_url = '/api/v1/jobs/%s' % job2_id
job2_set_status = httmock.urlmatch(
path=r'^%s$' % job2_status_update_url, method='PATCH')(_set_status)
with httmock.HTTMock(job1_get_status, job1_set_status,
job2_get_status, job2_set_status):
job.monitor_jobs(cluster, [job1_model, job2_model], **{'girder_token': 's', 'log_write_url': 1})
self.assertTrue(self._get_status_calls[job1_id])
self.assertTrue(self._set_status_calls[job1_id])
self.assertTrue(self._get_status_calls[job2_id])
self.assertTrue(self._set_status_calls[job2_id])
# All jobs are complete fo we shouldn't resheduled
self.assertFalse(retry.call_args_list)
from httmock import urlmatch
cim_url_match = urlmatch(scheme='https',
netloc=r'^api\.authorize\.net$',
path=r'^/xml/v1/request\.api$')
delete_success = (
'<?xml version="1.0" encoding="utf-8"?>'
'<{0}>'
'<messages>'
'<resultCode>Ok</resultCode>'
'<message><code>I00001</code><text>Successful.</text></message>'
'</messages>'
'</{0}>')
customer_profile_success = (
'<?xml version="1.0" encoding="utf-8"?>'
'<{0}>'
'<messages>'
'<resultCode>Ok</resultCode>'
'<message><code>I00001</code><text>Successful.</text></message>'
'</messages>'
'<customerProfileId>6666</customerProfileId>'
'<customerPaymentProfileIdList>'
'<numericString>7777</numericString>'
'</customerPaymentProfileIdList>'
'<customerShippingAddressIdList />'
'<validationDirectResponseList />'
'</{0}>')
payment_profile_success = (
def test_submit_job(self, get_connection, *args):
cluster = {
'_id': 'bob',
'type': 'ec2',
'name': 'dummy',
'config': {
'_id': 'dummy',
'scheduler': {
'type': 'sge'
}
},
}
job_id = 'dummy'
job_model = {
'_id': job_id,
'queueJobId': '1',
'name': 'dummy',
'commands': ['ls'],
'output': [{'tail': True, 'path': 'dummy/file/path'}]
}
qconf_output = ['pe_name orte',
'slots 10\n',
'user_lists NONE',
'xuser_lists NONE',
'start_proc_args /bin/true',
'stop_proc_args /bin/true',
'allocation_rule $pe_slots',
'control_slaves FALSE',
'job_is_first_task TRUE',
'urgency_slots min',
'accounting_summary FALSE']
qsub_output = ['Your job 74 ("test.sh") has been submitted']
conn = get_connection.return_value.__enter__.return_value
conn.execute.side_effect = [['/home/test'], qconf_output, qsub_output]
def _get_status(url, request):
content = {
'status': 'queued'
}
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
return httmock.response(200, content, headers, request=request)
def _set_status(url, request):
content = {
'status': 'queued'
}
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
return httmock.response(200, content, headers, request=request)
def _log(url, request):
content = {
}
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
return httmock.response(200, content, headers, request=request)
status_url = '/api/v1/jobs/%s/status' % job_id
get_status = httmock.urlmatch(
path=r'^%s$' % status_url, method='GET')(_get_status)
status_update_url = '/api/v1/jobs/%s' % job_id
set_status = httmock.urlmatch(
path=r'^%s$' % status_update_url, method='PATCH')(_set_status)
log_url = '/api/v1/jobs/%s/log' % job_id
log = httmock.urlmatch(
path=r'^%s$' % log_url, method='POST')(_log)
with httmock.HTTMock(get_status, set_status, log):
job.submit_job(cluster, job_model, log_write_url='log_write_url',
girder_token='girder_token')
self.assertEqual(conn.execute.call_args_list[1],
mock.call('qconf -sp orte'), 'Unexpected qconf command: %s' %
str(conn.execute.call_args_list[0]))
# Specifying and parallel environment
job_model = {
'_id': job_id,
'queueJobId': '1',
'name': 'dummy',
'commands': ['ls'],
'output': [{'tail': True, 'path': 'dummy/file/path'}],
'params': {
'parallelEnvironment': 'mype'
}
}
qconf_output = ['pe_name mype',
'slots 10\n',
'user_lists NONE',
'xuser_lists NONE',
'start_proc_args /bin/true',
'stop_proc_args /bin/true',
'allocation_rule $pe_slots',
'control_slaves FALSE',
'job_is_first_task TRUE',
'urgency_slots min',
'accounting_summary FALSE']
conn.reset_mock()
conn.execute.side_effect = [['/home/test'], qconf_output, qsub_output]
with httmock.HTTMock(get_status, set_status, log):
job.submit_job(cluster, job_model, log_write_url='log_write_url',
girder_token='girder_token')
self.assertEqual(conn.execute.call_args_list[1],
mock.call('qconf -sp mype'), 'Unexpected qconf command: %s' %
str(conn.execute.call_args_list[0]))
# For traditional clusters we shouldn't try to extract slot from orte
cluster = {
'_id': 'dummy',
'type': 'trad',
'name': 'dummy',
'config': {
'host': 'dummy',
'ssh': {
'user': '******',
'passphrase': 'its a secret'
},
'scheduler': {
'type': 'sge'
}
}
}
job_id = 'dummy'
job_model = {
'_id': job_id,
'queueJobId': '1',
'name': 'dummy',
'commands': ['ls'],
'output': [{'tail': True, 'path': 'dummy/file/path'}]
}
conn.reset_mock()
conn.execute.side_effect = [['/home/test'], ['Your job 74 ("test.sh") has been submitted']]
with httmock.HTTMock(get_status, set_status, log):
job.submit_job(cluster, job_model, log_write_url='log_write_url',
girder_token='girder_token')
# Assert that we don't try and get the number of slots
self.assertFalse('qconf' in str(conn.execute.call_args_list), 'qconf should not be called')
# For traditional clusters define a parallel env
cluster = {
'_id': 'dummy',
'type': 'trad',
'name': 'dummy',
'config': {
'host': 'dummy',
'ssh': {
'user': '******',
'passphrase': 'its a secret'
},
'scheduler': {
'type': 'sge'
}
}
}
job_id = 'dummy'
job_model = {
'_id': job_id,
'queueJobId': '1',
'name': 'dummy',
'commands': ['ls'],
'output': [{'tail': True, 'path': 'dummy/file/path'}],
'params': {
'parallelEnvironment': 'mype'
}
}
conn.reset_mock()
conn.execute.side_effect = [['/home/test'], qconf_output, ['Your job 74 ("test.sh") has been submitted']]
with httmock.HTTMock(get_status, set_status, log):
job.submit_job(cluster, job_model, log_write_url='log_write_url',
girder_token='girder_token')
self.assertEqual(conn.execute.call_args_list[1], mock.call('qconf -sp mype'))
self.assertEqual(job_model['params']['numberOfSlots'], 10)
def test_monitor_job_tail_output(self, get_connection, retry, *args):
job_id = 'dummy'
cluster = {
'_id': 'bill',
'type': 'ec2',
'name': 'dummy',
'config': {
'_id': 'dummy',
'scheduler': {
'type': 'sge'
}
}
}
job_model = {
'_id': job_id,
'queueJobId': '1',
'name': 'dummy',
'output': [{
'tail': True,
'path': 'dummy/file/path'
}],
'dir': '/home/test'
}
conn = get_connection.return_value.__enter__.return_value
conn.execute.side_effect = [
'job-ID prior name user state submit/start at queue slots ja-task-ID',
'-----------------------------------------------------------------------------------------',
'1 0.00000 hostname sgeadmin r 09/09/2009 14:58:14 1'
], ['i have a tail', 'asdfas']
def _get_status(url, request):
content = {'status': 'running'}
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
self._get_status_called = True
return httmock.response(200, content, headers, request=request)
def _set_status(url, request):
expected = {
u'status':
u'running',
u'output': [{
u'content': [u'i have a tail', u'asdfas'],
u'path': u'dummy/file/path',
u'tail': True
}],
u'timings': {}
}
self._set_status_called = json.loads(
request.body.decode('utf8')) == expected
if not self._set_status_called:
six.print_(json.loads(request.body.decode('utf8')),
file=sys.stderr)
return httmock.response(200, None, {}, request=request)
status_url = '/api/v1/jobs/%s/status' % job_id
get_status = httmock.urlmatch(path=r'^%s$' % status_url,
method='GET')(_get_status)
status_update_url = '/api/v1/jobs/%s' % job_id
set_status = httmock.urlmatch(path=r'^%s$' % status_update_url,
method='PATCH')(_set_status)
with httmock.HTTMock(get_status, set_status):
job.monitor_job(cluster, job_model, **{
'girder_token': 's',
'log_write_url': 1
})
self.assertTrue(self._get_status_called,
'Expect get status endpoint to be hit')
self.assertTrue(self._set_status_called,
'Expect set status endpoint to be hit')
def common_interceptor(callback, **kwargs):
decorator = urlmatch(**kwargs) if kwargs else all_requests
mock = decorator(callback)
return HTTMock(mock)
from httmock import urlmatch
cim_url_match = urlmatch(scheme='https', netloc=r'^api\.authorize\.net$',
path=r'^/xml/v1/request\.api$')
delete_success = (
'<?xml version="1.0" encoding="utf-8"?>'
'<{0}>'
'<messages>'
'<resultCode>Ok</resultCode>'
'<message><code>I00001</code><text>Successful.</text></message>'
'</messages>'
'</{0}>'
)
customer_profile_success = (
'<?xml version="1.0" encoding="utf-8"?>'
'<{0}>'
'<messages>'
'<resultCode>Ok</resultCode>'
'<message><code>I00001</code><text>Successful.</text></message>'
'</messages>'
'<customerProfileId>6666</customerProfileId>'
'<customerPaymentProfileIdList>'
'<numericString>7777</numericString>'
'</customerPaymentProfileIdList>'
'<customerShippingAddressIdList />'
'<validationDirectResponseList />'
def test_monitor_jobs_queued_no_retry(self, retry, get_connection):
conn = get_connection.return_value.__enter__.return_value
cluster = {
'_id': 'lost',
'type': 'ec2',
'name': 'dummy',
'config': {
'_id': 'dummy',
'scheduler': {
'type': 'sge'
}
}
}
job1_id = 'dummy1'
job1_model = {
'_id': job1_id,
'queueJobId': '1',
'name': 'dummy',
'output': []
}
job2_id = 'dummy1'
job2_model = {
'_id': job2_id,
'queueJobId': '2',
'name': 'dummy',
'output': []
}
conn.execute.return_value = [
'job-ID prior name user state submit/start at queue slots ja-task-ID',
'-----------------------------------------------------------------------------------------'
]
self._get_status_calls = {}
self._set_status_calls = {}
def _get_status(url, request):
content = {'status': 'complete'}
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
job_id = url.path.split('/')[-2]
self._get_status_calls[job_id] = True
return httmock.response(200, content, headers, request=request)
def _set_status(url, request):
expected = {'status': 'complete', 'timings': {}, 'output': []}
job_id = url.path.split('/')[-1]
self._set_status_calls[job_id] = True
self._set_status_called = json.loads(
request.body.decode('utf8')) == expected
return httmock.response(200, None, {}, request=request)
job1_status_url = '/api/v1/jobs/%s/status' % job1_id
job1_get_status = httmock.urlmatch(path=r'^%s$' % job1_status_url,
method='GET')(_get_status)
job1_status_update_url = '/api/v1/jobs/%s' % job1_id
job1_set_status = httmock.urlmatch(path=r'^%s$' %
job1_status_update_url,
method='PATCH')(_set_status)
job2_status_url = '/api/v1/jobs/%s/status' % job2_id
job2_get_status = httmock.urlmatch(path=r'^%s$' % job2_status_url,
method='GET')(_get_status)
job2_status_update_url = '/api/v1/jobs/%s' % job2_id
job2_set_status = httmock.urlmatch(path=r'^%s$' %
job2_status_update_url,
method='PATCH')(_set_status)
with httmock.HTTMock(job1_get_status, job1_set_status, job2_get_status,
job2_set_status):
job.monitor_jobs(cluster, [job1_model, job2_model], **{
'girder_token': 's',
'log_write_url': 1
})
self.assertTrue(self._get_status_calls[job1_id])
self.assertTrue(self._set_status_calls[job1_id])
self.assertTrue(self._get_status_calls[job2_id])
self.assertTrue(self._set_status_calls[job2_id])
# All jobs are complete fo we shouldn't resheduled
self.assertFalse(retry.call_args_list)
def test_monitor_job_terminated(self, get_connection):
conn = get_connection.return_value.__enter__.return_value
job_id = 'dummy'
cluster = {
'_id': 'bob',
'type': 'ec2',
'name': 'dummy',
'config': {
'_id': 'dummy',
'scheduler': {
'type': 'sge'
}
}
}
job_model = {
'_id': job_id,
'queueJobId': 'dummy',
'name': 'dummy',
'output': []
}
conn.execute.return_value = 'qstat output'
def _get_status(url, request):
content = {'status': 'terminating'}
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
self._get_status_called = True
return httmock.response(200, content, headers, request=request)
def _set_status(url, request):
expected = {
u'status': u'terminated',
u'timings': {},
u'output': []
}
self._set_status_called = json.loads(
request.body.decode('utf8')) == expected
return httmock.response(200, None, {}, request=request)
status_url = '/api/v1/jobs/%s/status' % job_id
get_status = httmock.urlmatch(path=r'^%s$' % status_url,
method='GET')(_get_status)
status_update_url = '/api/v1/jobs/%s' % job_id
set_status = httmock.urlmatch(path=r'^%s$' % status_update_url,
method='PATCH')(_set_status)
with httmock.HTTMock(get_status, set_status):
job.monitor_job(cluster, job_model, **{
'girder_token': 's',
'log_write_url': 1
})
self.assertTrue(self._get_status_called,
'Expect get status endpoint to be hit')
self.assertTrue(self._set_status_called,
'Expect set status endpoint to be hit')
def test_submit_job(self, get_connection, *args):
cluster = {
'_id': 'bob',
'type': 'ec2',
'name': 'dummy',
'config': {
'_id': 'dummy',
'scheduler': {
'type': 'sge'
}
},
}
job_id = 'dummy'
job_model = {
'_id': job_id,
'queueJobId': '1',
'name': 'dummy',
'commands': ['ls'],
'output': [{
'tail': True,
'path': 'dummy/file/path'
}]
}
qconf_output = [
'pe_name orte', 'slots 10\n',
'user_lists NONE', 'xuser_lists NONE',
'start_proc_args /bin/true', 'stop_proc_args /bin/true',
'allocation_rule $pe_slots', 'control_slaves FALSE',
'job_is_first_task TRUE', 'urgency_slots min',
'accounting_summary FALSE'
]
qsub_output = ['Your job 74 ("test.sh") has been submitted']
conn = get_connection.return_value.__enter__.return_value
conn.execute.side_effect = [['/home/test'], qconf_output, qsub_output]
def _get_status(url, request):
content = {'status': 'queued'}
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
return httmock.response(200, content, headers, request=request)
def _set_status(url, request):
content = {'status': 'queued'}
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
return httmock.response(200, content, headers, request=request)
def _log(url, request):
content = {}
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
return httmock.response(200, content, headers, request=request)
status_url = '/api/v1/jobs/%s/status' % job_id
get_status = httmock.urlmatch(path=r'^%s$' % status_url,
method='GET')(_get_status)
status_update_url = '/api/v1/jobs/%s' % job_id
set_status = httmock.urlmatch(path=r'^%s$' % status_update_url,
method='PATCH')(_set_status)
log_url = '/api/v1/jobs/%s/log' % job_id
log = httmock.urlmatch(path=r'^%s$' % log_url, method='POST')(_log)
with httmock.HTTMock(get_status, set_status, log):
job.submit_job(cluster,
job_model,
log_write_url='log_write_url',
girder_token='girder_token')
self.assertEqual(
conn.execute.call_args_list[1], mock.call('qconf -sp orte'),
'Unexpected qconf command: %s' %
str(conn.execute.call_args_list[0]))
# Specifying and parallel environment
job_model = {
'_id': job_id,
'queueJobId': '1',
'name': 'dummy',
'commands': ['ls'],
'output': [{
'tail': True,
'path': 'dummy/file/path'
}],
'params': {
'parallelEnvironment': 'mype'
}
}
qconf_output = [
'pe_name mype', 'slots 10\n',
'user_lists NONE', 'xuser_lists NONE',
'start_proc_args /bin/true', 'stop_proc_args /bin/true',
'allocation_rule $pe_slots', 'control_slaves FALSE',
'job_is_first_task TRUE', 'urgency_slots min',
'accounting_summary FALSE'
]
conn.reset_mock()
conn.execute.side_effect = [['/home/test'], qconf_output, qsub_output]
with httmock.HTTMock(get_status, set_status, log):
job.submit_job(cluster,
job_model,
log_write_url='log_write_url',
girder_token='girder_token')
self.assertEqual(
conn.execute.call_args_list[1], mock.call('qconf -sp mype'),
'Unexpected qconf command: %s' %
str(conn.execute.call_args_list[0]))
# For traditional clusters we shouldn't try to extract slot from orte
cluster = {
'_id': 'dummy',
'type': 'trad',
'name': 'dummy',
'config': {
'host': 'dummy',
'ssh': {
'user': '******',
'passphrase': 'its a secret'
},
'scheduler': {
'type': 'sge'
}
}
}
job_id = 'dummy'
job_model = {
'_id': job_id,
'queueJobId': '1',
'name': 'dummy',
'commands': ['ls'],
'output': [{
'tail': True,
'path': 'dummy/file/path'
}]
}
conn.reset_mock()
conn.execute.side_effect = [[
'/home/test'
], ['Your job 74 ("test.sh") has been submitted']]
with httmock.HTTMock(get_status, set_status, log):
job.submit_job(cluster,
job_model,
log_write_url='log_write_url',
girder_token='girder_token')
# Assert that we don't try and get the number of slots
self.assertFalse('qconf' in str(conn.execute.call_args_list),
'qconf should not be called')
# For traditional clusters define a parallel env
cluster = {
'_id': 'dummy',
'type': 'trad',
'name': 'dummy',
'config': {
'host': 'dummy',
'ssh': {
'user': '******',
'passphrase': 'its a secret'
},
'scheduler': {
'type': 'sge'
}
}
}
job_id = 'dummy'
job_model = {
'_id': job_id,
'queueJobId': '1',
'name': 'dummy',
'commands': ['ls'],
'output': [{
'tail': True,
'path': 'dummy/file/path'
}],
'params': {
'parallelEnvironment': 'mype'
}
}
conn.reset_mock()
conn.execute.side_effect = [[
'/home/test'
], qconf_output, ['Your job 74 ("test.sh") has been submitted']]
with httmock.HTTMock(get_status, set_status, log):
job.submit_job(cluster,
job_model,
log_write_url='log_write_url',
girder_token='girder_token')
self.assertEqual(conn.execute.call_args_list[1],
mock.call('qconf -sp mype'))
self.assertEqual(job_model['params']['numberOfSlots'], 10)
def test_monitor_job_complete(self, get_connection):
conn = get_connection.return_value.__enter__.return_value
conn.stat.return_value.st_size = 0
job_id = 'dummy'
cluster = {
'_id': 'dummy',
'type': 'ec2',
'name': 'dummy',
'config': {
'_id': 'dummy',
'scheduler': {
'type': 'sge'
}
}
}
job_model = {
'_id': job_id,
'queueJobId': 'dummy',
'name': 'dummy',
'output': [{
'itemId': 'dummy'
}],
'dir': '/home/test/%s' % job_id
}
conn.execute.return_value = 'qstat output'
def _get_status(url, request):
content = {
'status': 'running'
}
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
self._get_status_called = True
return httmock.response(200, content, headers, request=request)
def _set_status(url, request):
expected = {'status': 'uploading', 'timings': {}, 'output': [{'itemId': u'dummy'}]}
self._set_status_called = json.loads(request.body) == expected
return httmock.response(200, None, {}, request=request)
status_url = '/api/v1/jobs/%s/status' % job_id
get_status = httmock.urlmatch(
path=r'^%s$' % status_url, method='GET')(_get_status)
status_update_url = '/api/v1/jobs/%s' % job_id
set_status = httmock.urlmatch(
path=r'^%s$' % status_update_url, method='PATCH')(_set_status)
with httmock.HTTMock(get_status, set_status):
job.monitor_job(cluster, job_model, **{'girder_token': 's', 'log_write_url': 1})
self.assertTrue(self._get_status_called, 'Expect get status endpoint to be hit')
self.assertTrue(self._set_status_called, 'Expect set status endpoint to be hit')
expected_calls = [[[{u'config': {u'_id': u'dummy', u'scheduler': {u'type': u'sge'}}, u'name': u'dummy', u'type': u'ec2', u'_id': u'dummy'}, {u'status': u'uploading', u'output': [{u'itemId': u'dummy'}], u'_id': u'dummy', u'queueJobId': u'dummy', u'name': u'dummy', u'dir': u'/home/test/dummy'}], {u'girder_token': u's', u'log_write_url': 1, u'job_dir': u'/home/test/dummy'}]]
self.assertCalls(self._upload_job_output.call_args_list, expected_calls)
def test_connection(self, get_connection):
def valid(self):
return True
cluster_id = 'dummy_id'
cluster_model = {
'type': 'trad',
'name': 'my trad cluster',
'config': {
'conn': {
'user': '******'
},
'host': 'myhost'
},
'_id': cluster_id
}
self._set_call_value_index = 0
def _set_status(url, request):
expected = {'status': self._expected_status}
self._set_status_called = True
self._set_status_valid = json.loads(request.body) == expected
self._set_status_request = request.body
return httmock.response(200, None, {}, request=request)
status_update_url = '/api/v1/clusters/%s' % cluster_id
set_status = httmock.urlmatch(
path=r'^%s$' % status_update_url, method='PATCH')(_set_status)
def _log(url, request):
return httmock.response(200, None, {}, request=request)
log_url = '/api/v1/clusters/%s/log' % cluster_id
log = httmock.urlmatch(
path=r'^%s$' % log_url, method='POST')(_log)
with httmock.HTTMock(set_status, log):
cluster.test_connection(cluster_model, **{'girder_token': 's'})
self.assertTrue(self._set_status_called, 'Set status endpoint not called')
self.assertTrue(self._set_status_valid,
'Set status endpoint called with incorrect content: %s'
% self._set_status_request)
# Mock our conn calls and try again
def _get_cluster(url, request):
content = {
"_id": "55ef53bff657104278e8b185",
"config": {
"host": "ulmus",
"conn": {
"publicKey": "conn-rsa dummy",
'passphrase': 'dummy',
"user": "******"
}
}
}
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
return httmock.response(200, content, headers, request=request)
cluster_url = '/api/v1/clusters/%s' % cluster_id
get_cluster = httmock.urlmatch(
path=r'^%s$' % cluster_url, method='GET')(_get_cluster)
conn = get_connection.return_value.__enter__.return_value
conn.execute.return_value = ['/usr/bin/qsub']
self._expected_status = 'running'
with httmock.HTTMock(set_status, get_cluster):
cluster.test_connection(cluster_model, **{'girder_token': 's', 'log_write_url': 'http://localhost/log'})
self.assertTrue(self._set_status_called, 'Set status endpoint not called')
self.assertTrue(self._set_status_valid,
'Set status endpoint called in incorrect content: %s'
% self._set_status_request)
def test_import_path(self):
file = {
'name': 'test.txt',
'mode': 1,
'size': 123
}
folder = {
'name': 'folder',
'mode': stat.S_IFDIR,
'size': 1234
}
cluster_connection = mock.MagicMock()
cluster_connection.list.side_effect = [[file, folder], [file]]
girder_token = 'dummy'
parent = {
'_id': 'dummy_id'
}
path = '/my/path'
assetstore_id = 'dummy_id'
# Mock create item
def _create_item(url, request):
content = {
'_id': 'dummy'
}
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
self._item_requests.append(request)
return httmock.response(200, content, headers, request=request)
item_url = '/api/v1/item'
create_item = httmock.urlmatch(
path=r'^%s$' % item_url, method='POST')(_create_item)
# Mock create file
def _create_file(url, request):
content = {
'_id': 'dummy'
}
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
self._file_requests.append(request)
return httmock.response(200, content, headers, request=request)
file_url = '/api/v1/sftp_assetstores/%s/files' % assetstore_id
create_file = httmock.urlmatch(
path=r'^%s$' % file_url, method='POST')(_create_file)
# Mock create folder
def _create_folder(url, request):
content = {
'_id': 'dummy'
}
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
self._folder_requests.append(request)
return httmock.response(200, content, headers, request=request)
folder_url = '/api/v1/folder'
create_folder = httmock.urlmatch(
path=r'^%s$' % folder_url, method='POST')(_create_folder)
# Mock list folder
def _list_folder(url, request):
content = []
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
return httmock.response(200, content, headers, request=request)
folder_url = '/api/v1/folder'
list_folder = httmock.urlmatch(
path=r'^%s$' % folder_url, method='GET')(_list_folder)
with httmock.HTTMock(create_item, create_file, create_folder, list_folder):
download_path(cluster_connection, girder_token, parent, path,
'sftp_assetstores', assetstore_id, upload=False)
self.assertEqual(len(self._item_requests), 2)
self.assertEqual(len(self._file_requests), 2)
self.assertEqual(len(self._folder_requests), 1)
def test_connection(self, get_connection):
def valid(self):
return True
cluster_id = 'dummy_id'
cluster_model = {
'type': 'trad',
'name': 'my trad cluster',
'config': {
'conn': {
'user': '******'
},
'host': 'myhost'
},
'_id': cluster_id
}
self._set_call_value_index = 0
def _set_status(url, request):
expected = {'status': self._expected_status}
self._set_status_called = True
self._set_status_valid = json.loads(request.body.decode('utf8')) == expected
self._set_status_request = request.body.decode('utf8')
return httmock.response(200, None, {}, request=request)
status_update_url = '/api/v1/clusters/%s' % cluster_id
set_status = httmock.urlmatch(
path=r'^%s$' % status_update_url, method='PATCH')(_set_status)
def _log(url, request):
return httmock.response(200, None, {}, request=request)
log_url = '/api/v1/clusters/%s/log' % cluster_id
log = httmock.urlmatch(
path=r'^%s$' % log_url, method='POST')(_log)
with httmock.HTTMock(set_status, log):
cluster.test_connection(cluster_model, **{'girder_token': 's'})
self.assertTrue(self._set_status_called, 'Set status endpoint not called')
self.assertTrue(self._set_status_valid,
'Set status endpoint called with incorrect content: %s'
% self._set_status_request)
# Mock our conn calls and try again
def _get_cluster(url, request):
content = {
"_id": "55ef53bff657104278e8b185",
"config": {
"host": "ulmus",
"conn": {
"publicKey": "conn-rsa dummy",
'passphrase': 'dummy',
"user": "******"
}
}
}
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
return httmock.response(200, content, headers, request=request)
cluster_url = '/api/v1/clusters/%s' % cluster_id
get_cluster = httmock.urlmatch(
path=r'^%s$' % cluster_url, method='GET')(_get_cluster)
conn = get_connection.return_value.__enter__.return_value
conn.execute.return_value = ['/usr/bin/qsub']
self._expected_status = 'running'
with httmock.HTTMock(set_status, get_cluster):
cluster.test_connection(cluster_model, **{'girder_token': 's', 'log_write_url': 'http://localhost/log'})
self.assertTrue(self._set_status_called, 'Set status endpoint not called')
self.assertTrue(self._set_status_valid,
'Set status endpoint called in incorrect content: %s'
% self._set_status_request)
def test_monitor_job_complete(self, get_connection):
conn = get_connection.return_value.__enter__.return_value
conn.stat.return_value.st_size = 0
job_id = 'dummy'
cluster = {
'_id': 'dummy',
'type': 'ec2',
'name': 'dummy',
'config': {
'_id': 'dummy',
'scheduler': {
'type': 'sge'
}
}
}
job_model = {
'_id': job_id,
'queueJobId': 'dummy',
'name': 'dummy',
'output': [{
'itemId': 'dummy'
}],
'dir': '/home/test/%s' % job_id
}
conn.execute.return_value = 'qstat output'
def _get_status(url, request):
content = {'status': 'running'}
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
self._get_status_called = True
return httmock.response(200, content, headers, request=request)
def _set_status(url, request):
expected = {
'status': 'uploading',
'timings': {},
'output': [{
'itemId': u'dummy'
}]
}
print('set_status')
self._set_status_called = json.loads(
request.body.decode('utf8')) == expected
return httmock.response(200, None, {}, request=request)
status_url = '/api/v1/jobs/%s/status' % job_id
get_status = httmock.urlmatch(path=r'^%s$' % status_url,
method='GET')(_get_status)
status_update_url = '/api/v1/jobs/%s' % job_id
set_status = httmock.urlmatch(path=r'^%s$' % status_update_url,
method='PATCH')(_set_status)
with httmock.HTTMock(get_status, set_status):
job.monitor_job(cluster, job_model, **{
'girder_token': 's',
'log_write_url': 1
})
self.assertTrue(self._get_status_called,
'Expect get status endpoint to be hit')
self.assertTrue(self._set_status_called,
'Expect set status endpoint to be hit')
expected_calls = [[[{
u'config': {
u'_id': u'dummy',
u'scheduler': {
u'type': u'sge'
}
},
u'name': u'dummy',
u'type': u'ec2',
u'_id': u'dummy'
}, {
u'status': u'uploading',
u'output': [{
u'itemId': u'dummy'
}],
u'_id': u'dummy',
u'queueJobId': u'dummy',
u'name': u'dummy',
u'dir': u'/home/test/dummy'
}], {
u'girder_token': u's',
u'log_write_url': 1,
u'job_dir': u'/home/test/dummy'
}]]
self.assertCalls(self._upload_job_output.call_args_list,
expected_calls)