def side_effect(*args, **kwargs):
if "metadata" in args[0]:
return mock_response(json={
"type": "file",
"content": "%s/%s" % (self.datanode_base, os.path.basename(file_name))
})
if "content" in args[0]:
content_length, chunk_size = len(content), len(content)/max_chunk_size
chunks = [content[i:i+chunk_size] for i in range(0, content_length, chunk_size)]
return mock_response(chunks=chunks)
def test_list_services(self, getMock):
result = ['PIG', 'OOZIE']
getMock.return_value = mock_response(status_code=200, json=result)
rep = self.__create_proto().list_services()
self.assert_called_once_with(getMock, self.api_url + '/services')
self.assertEquals(rep, result)
def test_list_clusters(self, getMock):
getMock.return_value = mock_response(status_code=200,
json={
'clusters': [{
'id': '1',
'name': 'cluster1',
'state': 'ready'
}, {
'id':
'2',
'name':
'cluster2',
'state':
'terminated'
}]
})
rep = self.__create_proto().get_clusters()
clusters = rep['clusters']
self.assert_called_once_with(getMock, self.api_url + '/cluster')
self.assertEquals(len(clusters), 2)
self.assertEquals(clusters[0]['id'], '1')
self.assertEquals(clusters[0]['name'], 'cluster1')
self.assertEquals(clusters[0]['state'], 'ready')
self.assertEquals(clusters[1]['id'], '2')
self.assertEquals(clusters[1]['name'], 'cluster2')
self.assertEquals(clusters[1]['state'], 'terminated')
def test_get_nonexisting_file(self):
self.client.get.return_value = mock_response(status_code=404)
out_file = StringIO("")
self.assertRaisesRegexp(
ResponseError, 'File /file.txt does not exist',
self.instance.get_file, '/file.txt', out_file)
out_file.close()
def test_get_nonexisting_file(self):
self.client.get.return_value = mock_response(status_code=404)
out_file = StringIO("")
self.assertRaisesRegexp(OperationError,
'File /file.txt does not exist',
self.instance.get_file, '/file.txt', out_file)
out_file.close()
def test_delete_path(self):
self.client.delete.return_value = mock_response(status_code=204)
self.instance.delete_path('/remote/file.txt')
self.client.delete.assert_called_with(
self.namenode_base + '/remote/file.txt',
auth=self.auth,
params={'recursive': 'false'})
def test_create_cluster_fail(self, postMock):
postMock.return_value = mock_response(
status_code=500,
json={'error': 'request failed due to server error'})
with self.assertRaises(ResponseError):
self.__create_proto().create_cluster('cluster1', 2, ['FOO', 'BAR'],
True)
def test_refuse_ssh_cluster_for_error_statuses(self):
response = mock_response(status_code=500)
with patch('requests.get', MagicMock(return_value=response)), \
mock_home():
self.assertRaisesRegexp(ResponseError,
'Cannot get cluster details', ssh_cluster,
'cluster1', self.config)
def test_delete_path(self):
self.client.delete.return_value = mock_response(status_code=204)
self.instance.delete_path('/remote/file.txt')
self.client.delete.assert_called_with(self.namenode_base +
'/remote/file.txt',
auth=self.auth,
params={'recursive': 'false'})
def test_get_file(self):
self.client.get.return_value = mock_response(raw=StringIO("hello"))
out_file = StringIO("")
size = self.instance.get_file('/remote/file.txt', out_file)
self.assertEquals(5, size)
self.assertEquals("hello", out_file.getvalue())
out_file.close()
def test_refuse_ssh_cluster_for_error_statuses(self):
response = mock_response(status_code=500)
with patch('requests.get', MagicMock(return_value=response)), \
mock_home():
self.assertRaisesRegexp(
ResponseError, 'Cannot get cluster details',
ssh_cluster, 'cluster1', self.config)
def test_refuse_ssh_nonexistent_cluster(self):
response = mock_response(status_code=404)
with patch('requests.get', MagicMock(return_value=response)), \
mock_home():
self.assertRaisesRegexp(
ExitWithError, 'cluster1 does not exist',
ssh_cluster, 'cluster1', self.config)
def test_list_services(self, getMock):
result = ['PIG', 'OOZIE']
getMock.return_value = mock_response(status_code=200,
json=result)
rep = self.__create_proto().list_services()
self.assert_called_once_with(getMock, self.api_url + '/services')
self.assertEquals(rep, result)
def test_refuse_ssh_clusters_in_post_running_states(self):
for state in ['terminating', 'terminated', 'failed']:
response = mock_response(json=dict(state=state))
with patch('requests.get', MagicMock(return_value=response)), \
mock_home():
self.assertRaisesRegexp(
ExitWithError, 'cluster in %s state' % state,
ssh_cluster, 'cluster1', self.config)
def test_delete_path(self):
self.client.delete.return_value = mock_response(json={
"boolean": True
})
self.assertTrue(self.instance.delete_path('/remote/file.txt'))
self.client.delete.assert_called_with(
self.namenode_base + '/remote/file.txt',
params={'user.name': 'user1', 'op': 'DELETE', 'recursive': 'false'})
def test_chmod(self):
self.client.post.return_value = mock_response(status_code=204)
self.instance.chmod('/some/path', '777')
self.client.post.assert_called_with(
self.namenode_base + '/some/path',
auth=self.auth,
data=json.dumps({'action': 'chmod', 'permissions': '777'})
)
def test_exit_with_error_when_ssh_command_is_not_executable(self):
response = mock_response(json=MagicMock(side_effect=[RUNNING, PROFILE]))
call_mock = MagicMock(side_effect=OSError())
with patch('requests.get', MagicMock(return_value=response)), \
patch('subprocess.call', call_mock), mock_home():
self.assertRaisesRegexp(ExitWithError, 'Cannot execute',
ssh_cluster, 'cluster1', self.config)
self.assertEmptyIterator(response.json.side_effect)
def test_refuse_ssh_clusters_in_post_running_states(self):
for state in ['terminating', 'terminated', 'failed']:
response = mock_response(json=dict(state=state))
with patch('requests.get', MagicMock(return_value=response)), \
mock_home():
self.assertRaisesRegexp(ExitWithError,
'cluster in %s state' % state,
ssh_cluster, 'cluster1', self.config)
def test_terminate_cluster(self, postMock):
postMock.return_value = mock_response(
status_code=200, json={'message': 'termination accepted'})
rep = self.__create_proto().terminate_cluster('1')
self.assert_called_once_with(postMock,
self.api_url + '/cluster/1/terminate')
self.assertEquals(rep['message'], 'termination accepted')
def test_format_webhdfs_error_messages(self):
ex = ResponseError("message", mock_response(json={
'RemoteException': {
'message': 'long Java stacktrace'
},
'IrrelevantStuff': 'skipped'
}))
self.assertIn('long Java stacktrace', str(ex))
self.assertNotIn('skipped', str(ex))
def test_exit_with_error_when_ssh_command_is_not_executable(self):
response = mock_response(json=MagicMock(
side_effect=[RUNNING, PROFILE]))
call_mock = MagicMock(side_effect=OSError())
with patch('requests.get', MagicMock(return_value=response)), \
patch('subprocess.call', call_mock), mock_home():
self.assertRaisesRegexp(ExitWithError, 'Cannot execute',
ssh_cluster, 'cluster1', self.config)
self.assertEmptyIterator(response.json.side_effect)
def side_effect(*args, **kwargs):
if "metadata" in args[0]:
return mock_response(
json={
"type":
"file",
"content":
"%s/%s" %
(self.datanode_base, os.path.basename(file_name))
})
if "content" in args[0]:
content_length, chunk_size = len(
content), len(content) / max_chunk_size
chunks = [
content[i:i + chunk_size]
for i in range(0, content_length, chunk_size)
]
return mock_response(chunks=chunks)
def test_connect(self):
response = mock_response(json={
'location': 'infinity://host:8080/',
'user': '******'
})
with patch('requests.get', MagicMock(return_value=response)):
result = connect(API_KEY, API_SECRET, API_URL)
client = result._StorageConnection__client
self.assertEquals(client.infinity_uri, 'infinity://host:8080/')
self.assertEquals(client.username, 'username')
def test_list_path(self):
statuses = [{"path": "a.txt"}, {"path": "b.txt"}]
self.client.get.return_value = mock_response(json={
"type": "directory",
"content": statuses
})
self.assertEquals(
self.instance.list_path('/some/path').statuses, statuses)
self.client.get.assert_called_with(self.namenode_base + '/some/path',
auth=self.auth)
def test_poll_service_until_provisioned(self):
response = mock_response(json=MagicMock(side_effect=[
PROVISIONING, PROVISIONING, PROVISIONING, RUNNING, PROFILE]))
call_mock = MagicMock(return_value=0)
with mock_home() as home, collect_outputs(), \
patch('requests.get', MagicMock(return_value=response)), \
patch('subprocess.call', call_mock), patch('time.sleep'):
self.assertEquals(0, ssh_cluster('cluster1', self.config))
self.assertEquals(home.read_last_cluster(), "cluster1")
self.assertEmptyIterator(response.json.side_effect)
def test_connect(self):
response = mock_response(json={
'location': 'webhdfs://host:8080/',
'user': '******'
})
with patch('requests.get', MagicMock(return_value=response)):
result = connect(API_KEY, API_SECRET, API_URL)
client = result._StorageConnection__client
self.assertEquals(client.webhdfs_uri, 'webhdfs://host:8080/')
self.assertEquals(client.username, 'username')
def test_terminate_cluster(self, postMock):
postMock.return_value = mock_response(status_code=200, json={
'message': 'termination accepted'
})
rep = self.__create_proto().terminate_cluster('1')
self.assert_called_once_with(postMock,
self.api_url + '/cluster/1/terminate')
self.assertEquals(rep['message'], 'termination accepted')
def test_file_upload_with_replication_error(self):
with TempDirectory() as local_dir:
self.client.put = mock_response(status_code=500, json={
'RemoteException': {
'exception': 'ArrayIndexOutOfBoundsException'
}
})
local_file = local_dir.write('file.txt', 'contents')
self.assertRaisesRegexp(ResponseError, 'Not redirected',
self.instance.put_file, local_file,
'/remote/path')
def test_chmod(self):
self.client.post.return_value = mock_response(status_code=204)
self.instance.chmod('/some/path', '777')
self.client.post.assert_called_with(self.namenode_base + '/some/path',
auth=self.auth,
data=json.dumps({
'action':
'chmod',
'permissions':
'777'
}))
def test_poll_service_until_provisioned(self):
response = mock_response(json=MagicMock(side_effect=[
PROVISIONING, PROVISIONING, PROVISIONING, RUNNING, PROFILE
]))
call_mock = MagicMock(return_value=0)
with mock_home() as home, collect_outputs(), \
patch('requests.get', MagicMock(return_value=response)), \
patch('subprocess.call', call_mock), patch('time.sleep'):
self.assertEquals(0, ssh_cluster('cluster1', self.config))
self.assertEquals(home.read_last_cluster(), "cluster1")
self.assertEmptyIterator(response.json.side_effect)
def test_remove_user_from_cluster(self, postMock):
postMock.return_value = mock_response(status_code=200, json={
'message': 'user removed successfully'
})
rep = self.__create_proto().remove_user_from_cluster('1', 'jsmith')
expected_body = json.dumps({ 'user' : 'jsmith' })
self.assert_called_once_with(postMock,
self.api_url + '/cluster/1/remove_user',
expected_body)
self.assertEquals(rep['message'], 'user removed successfully')
def test_format_webhdfs_error_messages(self):
ex = ResponseError(
"message",
mock_response(
json={
'RemoteException': {
'message': 'long Java stacktrace'
},
'IrrelevantStuff': 'skipped'
}))
self.assertIn('long Java stacktrace', str(ex))
self.assertNotIn('skipped', str(ex))
def test_ssh_clusters_in_running_state(self):
response = mock_response(json=MagicMock(
side_effect=[RUNNING, PROFILE]))
call_mock = MagicMock(return_value=0)
with patch('requests.get', MagicMock(return_value=response)), \
patch('subprocess.call', call_mock), mock_home():
self.assertEquals(0, ssh_cluster('cluster1', self.config))
call_mock.assert_called_with([
'ssh', '192.168.20.18', '-l', 'user1', '-o',
'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no',
'-L1:192.168.20.18:1', '-L2:192.168.20.18:2', '-L3:192.168.20.18:3'
])
self.assertEmptyIterator(response.json.side_effect)
def test_get_cluster_details(self, getMock):
getMock.return_value = mock_response(status_code=200, json={
'id': '1',
'name': 'cluster1',
'state': 'ready'
})
rep = self.__create_proto().get_cluster_details('1')
self.assert_called_once_with(getMock, self.api_url + '/cluster/1')
self.assertEquals(rep['id'], '1')
self.assertEquals(rep['name'], 'cluster1')
self.assertEquals(rep['state'], 'ready')
def test_get_cluster_details(self, getMock):
getMock.return_value = mock_response(status_code=200,
json={
'id': '1',
'name': 'cluster1',
'state': 'ready'
})
rep = self.__create_proto().get_cluster_details('1')
self.assert_called_once_with(getMock, self.api_url + '/cluster/1')
self.assertEquals(rep['id'], '1')
self.assertEquals(rep['name'], 'cluster1')
self.assertEquals(rep['state'], 'ready')
def test_list_path(self):
statuses = [
{"path": "a.txt"},
{"path": "b.txt"}
]
self.client.get.return_value = mock_response(json={
"type": "directory",
"content": statuses
})
self.assertEquals(self.instance.list_path('/some/path').statuses,
statuses)
self.client.get.assert_called_with(
self.namenode_base + '/some/path',
auth=self.auth)
def test_ssh_clusters_in_running_state(self):
response = mock_response(json=MagicMock(side_effect=[RUNNING, PROFILE]))
call_mock = MagicMock(return_value=0)
with patch('requests.get', MagicMock(return_value=response)), \
patch('subprocess.call', call_mock), mock_home():
self.assertEquals(0, ssh_cluster('cluster1', self.config))
call_mock.assert_called_with(['ssh', '192.168.20.18',
'-l', 'user1',
'-o', 'UserKnownHostsFile=/dev/null',
'-o', 'StrictHostKeyChecking=no',
'-L1:192.168.20.18:1',
'-L2:192.168.20.18:2',
'-L3:192.168.20.18:3'])
self.assertEmptyIterator(response.json.side_effect)
def test_ssh_cluster_with_custom_key(self):
self.config.ssh_key = '~/.ssh/aws_key'
response = mock_response(json=MagicMock(
side_effect=[RUNNING, PROFILE]))
call_mock = MagicMock(return_value=0)
with patch('requests.get', MagicMock(return_value=response)), \
patch('subprocess.call', call_mock), mock_home():
self.assertEquals(0, ssh_cluster('cluster1', self.config))
expected_key_path = path.expanduser(self.config.ssh_key)
call_mock.assert_called_with([
'ssh', '192.168.20.18', '-l', 'user1', '-o',
'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no',
'-L1:192.168.20.18:1', '-L2:192.168.20.18:2',
'-L3:192.168.20.18:3', '-i', expected_key_path
])
def test_create_cluster(self, postMock):
postMock.return_value = mock_response(status_code=201, json={
'id': '1',
'name': 'cluster1',
'state': 'ready'
})
rep = self.__create_proto().create_cluster('cluster1', 2, ['FOO'])
expected_body = json.dumps({ 'name' : 'cluster1', 'size' : 2,
'optionalServices': ['FOO'] })
self.assert_called_once_with(postMock, self.api_url + '/cluster',
expected_body)
self.assertEquals(rep['id'], '1')
self.assertEquals(rep['name'], 'cluster1')
self.assertEquals(rep['state'], 'ready')
def test_list_path(self):
statuses = [
{"pathSuffix": "a.txt"},
{"pathSuffix": "b.txt"}
]
self.client.get.return_value = mock_response(json={
"FileStatuses": {
"FileStatus": statuses
}
})
self.assertEquals(self.instance.list_path('/some/path').statuses,
statuses)
self.client.get.assert_called_with(
self.namenode_base + '/some/path',
params={'user.name': 'user1', 'op': 'LISTSTATUS'})
def test_ssh_cluster_with_custom_key(self):
self.config.ssh_key = '~/.ssh/aws_key'
response = mock_response(json=MagicMock(side_effect=[RUNNING, PROFILE]))
call_mock = MagicMock(return_value=0)
with patch('requests.get', MagicMock(return_value=response)), \
patch('subprocess.call', call_mock), mock_home():
self.assertEquals(0, ssh_cluster('cluster1', self.config))
expected_key_path = path.expanduser(self.config.ssh_key)
call_mock.assert_called_with(['ssh', '192.168.20.18',
'-l', 'user1',
'-o', 'UserKnownHostsFile=/dev/null',
'-o', 'StrictHostKeyChecking=no',
'-L1:192.168.20.18:1',
'-L2:192.168.20.18:2',
'-L3:192.168.20.18:3',
'-i', expected_key_path])
def test_list_clusters(self, getMock):
getMock.return_value = mock_response(status_code=200, json={
'clusters': [
{ 'id': '1', 'name': 'cluster1', 'state': 'ready' },
{ 'id': '2', 'name': 'cluster2', 'state': 'terminated' }
]
})
rep = self.__create_proto().get_clusters()
clusters = rep['clusters']
self.assert_called_once_with(getMock, self.api_url + '/cluster')
self.assertEquals(len(clusters), 2)
self.assertEquals(clusters[0]['id'], '1')
self.assertEquals(clusters[0]['name'], 'cluster1')
self.assertEquals(clusters[0]['state'], 'ready')
self.assertEquals(clusters[1]['id'], '2')
self.assertEquals(clusters[1]['name'], 'cluster2')
self.assertEquals(clusters[1]['state'], 'terminated')
def test_file_upload(self):
with TempDirectory() as local_dir:
file_created_json = {
"path": "/user/user1/remote/path",
"metadata": "%s/remote/path" % self.namenode_base,
"content": "%s/remote/path" % self.datanode_base,
"type": "file",
"owner": "user1",
"group": "users",
"permissions": "640",
"size": 0,
"modificationTime": "2014-04-08T12:31:45+0100",
"accessTime": "2014-04-08T12:45:22+0100",
"blockSize": 67108864,
"replication": 3
}
file_creation = mock_response(status_code=201,
json=file_created_json)
data_saved = requests.Response()
data_saved.status_code = 204
self.client.post = MagicMock(side_effect=[file_creation])
self.client.put = MagicMock(side_effect=[data_saved])
local_file = local_dir.write('file.txt', 'contents')
self.instance.put_file(local_file, '/remote/path')
self.assertEqual(self.client.post.call_count, 1)
self.assertEqual(self.client.put.call_count, 1)
self.client.post.assert_any_call(self.namenode_base + '/remote',
auth=self.auth,
data=json.dumps({
"action":
"mkfile",
"name":
"path",
"permissions":
"640"
}))
self.client.put.assert_any_call(self.datanode_base +
'/remote/path',
auth=self.auth,
data=local_file)
def test_file_upload(self):
with TempDirectory() as local_dir:
file_created_json = {
"path": "/user/user1/remote/path",
"metadata": "%s/remote/path" % self.namenode_base,
"content": "%s/remote/path" % self.datanode_base,
"type": "file",
"owner": "user1",
"group": "users",
"permissions": "640",
"size": 0,
"modificationTime": "2014-04-08T12:31:45+0100",
"accessTime": "2014-04-08T12:45:22+0100",
"blockSize": 67108864,
"replication": 3
}
file_creation = mock_response(status_code=201, json=file_created_json)
data_saved = requests.Response()
data_saved.status_code = 204
self.client.post = MagicMock(side_effect=[file_creation])
self.client.put = MagicMock(side_effect=[data_saved])
local_file = local_dir.write('file.txt', 'contents')
self.instance.put_file(local_file, '/remote/path')
self.assertEqual(self.client.post.call_count, 1)
self.assertEqual(self.client.put.call_count, 1)
self.client.post.assert_any_call(
self.namenode_base + '/remote',
auth=self.auth,
data=json.dumps({
"action": "mkfile",
"name": "path",
"permissions": "640"
})
)
self.client.put.assert_any_call(
self.datanode_base + '/remote/path',
auth=self.auth,
data=local_file
)
def test_create_cluster(self, postMock):
postMock.return_value = mock_response(status_code=201,
json={
'id': '1',
'name': 'cluster1',
'state': 'ready',
'shared': False
})
rep = self.__create_proto().create_cluster('cluster1', 2, ['FOO'],
False)
expected_body = json.dumps({
'name': 'cluster1',
'size': 2,
'optionalServices': ['FOO'],
'shared': False
})
self.assert_called_once_with(postMock, self.api_url + '/cluster',
expected_body)
self.assertEquals(rep['id'], '1')
self.assertEquals(rep['name'], 'cluster1')
self.assertEquals(rep['state'], 'ready')
self.assertEquals(rep['shared'], False)
def test_list_clusters_fail(self, getMock):
getMock.return_value = mock_response(
status_code=500,
json={'error': 'request failed due to server error'})
with self.assertRaises(ResponseError):
self.__create_proto().get_clusters()
def test_connect_when_service_is_unavailable(self):
response = mock_response(status_code=503)
with patch('requests.get', MagicMock(return_value=response)):
self.assertRaises(ResponseError, connect, API_KEY, API_SECRET,
API_URL)
def test_refuse_ssh_nonexistent_cluster(self):
response = mock_response(status_code=404)
with patch('requests.get', MagicMock(return_value=response)), \
mock_home():
self.assertRaisesRegexp(ExitWithError, 'cluster1 does not exist',
ssh_cluster, 'cluster1', self.config)
def test_format_plain_text_error_messages(self):
ex = ResponseError('message', mock_response(text='plain error'))
self.assertIn('plain error', str(ex))
def test_format_non_authorization_errors(self):
ex = ResponseError("message", mock_response(status_code=415))
self.assertIn("HTTP error (415)", str(ex))
def test_format_simple_error_messages(self):
ex = ResponseError("message",
mock_response(json={'error': 'simple error'}))
self.assertIn('simple error', str(ex))
def test_keep_attributes(self):
message = "trying to foo"
response = mock_response()
ex = ResponseError(message, response)
self.assertEquals(message, ex.message)
self.assertEquals(response, ex.response)
def test_format_authorization_errors(self):
ex = ResponseError("message", mock_response(status_code=401))
self.assertIn("Unauthorized request", str(ex))
def test_delete_path_with_error(self):
self.client.delete.return_value = mock_response(status_code=403)
self.assertRaises(ResponseError, self.instance.delete_path,
'/remote/file.txt')
def test_cannot_show_general_info(self):
response = mock_response(status_code=500)
with patch('requests.get', MagicMock(return_value=response)):
self.assertRaisesRegexp(
ResponseError, 'Cannot get general information',
info_command, 'cluster1', self.config)
def test_create_cluster_fail(self, postMock):
postMock.return_value = mock_response(status_code=500, json={
'error': 'request failed due to server error'
})
with self.assertRaises(ResponseError):
self.__create_proto().create_cluster('cluster1', 2, ['FOO', 'BAR'])
def test_list_services_fail(self, getMock):
getMock.return_value = mock_response(status_code=404)
rep = self.__create_proto().list_services()
self.assert_called_once_with(getMock, self.api_url + '/services')
self.assertEquals(rep, [])
def test_remove_user_from_cluster_fail(self, postMock):
postMock.return_value = mock_response(status_code=500, json={
'error': 'request failed due to server error'
})
with self.assertRaises(ResponseError):
self.__create_proto().remove_user_from_cluster('1', 'jsmith')
def test_list_nonexistent_path(self):
self.client.get.return_value = mock_response(status_code=404)
listing = self.instance.list_path('/some/path')
self.assertIsNone(listing)
def test_get_cluster_details_fail(self, getMock):
getMock.return_value = mock_response(status_code=500, json={
'error': 'request failed due to server error'
})
with self.assertRaises(ResponseError):
self.__create_proto().get_cluster_details('1')
