def get_raw_binary(job_binary, proxy_configs=None,
with_context=False, remote=None):
'''Get the raw data for a job binary
This will retrieve the raw data for a job binary from it's source. In the
case of Swift based binaries there is a precedence of credentials for
authenticating the client. Requesting a context based authentication takes
precendence over proxy user which takes precendence over embedded
credentials.
:param job_binary: The job binary to retrieve
:param proxy_configs: Proxy user configuration to use as credentials
:param with_context: Use the current context as credentials
:param remote: The remote contains node group and cluster information
:returns: The raw data from a job binary
'''
url = job_binary.url
if url.startswith("internal-db://"):
res = db.get_raw_data(context.ctx(), job_binary)
if url.startswith(su.SWIFT_INTERNAL_PREFIX):
if with_context:
res = i_swift.get_raw_data_with_context(job_binary)
else:
res = i_swift.get_raw_data(job_binary, proxy_configs)
if url.startswith(m.MANILA_PREFIX):
res = manila.get_file_info(job_binary, remote)
return res
def get_raw_binary(job_binary,
proxy_configs=None,
with_context=False,
remote=None):
'''Get the raw data for a job binary
This will retrieve the raw data for a job binary from it's source. In the
case of Swift based binaries there is a precedence of credentials for
authenticating the client. Requesting a context based authentication takes
precedence over proxy user which takes precedence over embedded
credentials.
:param job_binary: The job binary to retrieve
:param proxy_configs: Proxy user configuration to use as credentials
:param with_context: Use the current context as credentials
:param remote: The remote contains node group and cluster information
:returns: The raw data from a job binary
'''
url = job_binary.url
if url.startswith("internal-db://"):
res = db.get_raw_data(context.ctx(), job_binary)
if url.startswith(su.SWIFT_INTERNAL_PREFIX):
if with_context:
res = i_swift.get_raw_data_with_context(job_binary)
else:
res = i_swift.get_raw_data(job_binary, proxy_configs)
if url.startswith(m.MANILA_PREFIX):
res = manila.get_file_info(job_binary, remote)
return res
def get_raw_binary(job_binary, proxy_configs=None):
url = job_binary.url
if url.startswith("internal-db://"):
res = db.get_raw_data(context.ctx(), job_binary)
if url.startswith(su.SWIFT_INTERNAL_PREFIX):
res = i_swift.get_raw_data(context.ctx(), job_binary, proxy_configs)
return res
def get_raw_binary(job_binary, proxy_configs=None):
url = job_binary.url
if url.startswith("internal-db://"):
res = db.get_raw_data(context.ctx(), job_binary)
if url.startswith(su.SWIFT_INTERNAL_PREFIX):
res = i_swift.get_raw_data(job_binary, proxy_configs)
return res
def get_raw_binary(job_binary):
url = job_binary.url
if url.startswith("internal-db://"):
res = db.get_raw_data(context.ctx(), job_binary)
# TODO(mattf): remove support for OLD_SWIFT_INTERNAL_PREFIX
if url.startswith(su.SWIFT_INTERNAL_PREFIX) or (
url.startswith(su.OLD_SWIFT_INTERNAL_PREFIX)):
res = i_swift.get_raw_data(context.ctx(), job_binary)
return res
def get_raw_binary(job_binary):
url = job_binary.url
if url.startswith("internal-db://"):
res = db.get_raw_data(context.ctx(), job_binary)
# TODO(mattf): remove support for OLD_SWIFT_INTERNAL_PREFIX
if url.startswith(su.SWIFT_INTERNAL_PREFIX) or (url.startswith(
su.OLD_SWIFT_INTERNAL_PREFIX)):
res = i_swift.get_raw_data(context.ctx(), job_binary)
return res
def test_get_raw_data(self, swift_client, _get_raw_data):
client_instance = mock.Mock()
swift_client.return_value = client_instance
job_binary = mock.Mock()
job_binary.url = 'swift://container/object'
# embedded credentials
job_binary.extra = dict(user='******', password='******')
i_s.get_raw_data(job_binary)
swift_client.assert_called_with(username='******', password='******')
_get_raw_data.assert_called_with(job_binary, client_instance)
# proxy configs should override embedded credentials
proxy_configs = dict(proxy_username='******',
proxy_password='******',
proxy_trust_id='proxytrust')
i_s.get_raw_data(job_binary, proxy_configs)
swift_client.assert_called_with(username='******',
password='******',
trust_id='proxytrust')
_get_raw_data.assert_called_with(job_binary, client_instance)
def test_get_raw_data(self, swift_client, _get_raw_data):
client_instance = mock.Mock()
swift_client.return_value = client_instance
job_binary = mock.Mock()
job_binary.url = 'swift://container/object'
# embedded credentials
job_binary.extra = dict(user='******', password='******')
i_s.get_raw_data(job_binary)
swift_client.assert_called_with(username='******',
password='******')
_get_raw_data.assert_called_with(job_binary, client_instance)
# proxy configs should override embedded credentials
proxy_configs = dict(proxy_username='******',
proxy_password='******',
proxy_trust_id='proxytrust')
i_s.get_raw_data(job_binary, proxy_configs)
swift_client.assert_called_with(username='******',
password='******',
trust_id='proxytrust')
_get_raw_data.assert_called_with(job_binary, client_instance)
def test_get_raw_data(self, swift_client):
client_instance = mock.Mock()
swift_client.return_value = client_instance
job_binary = mock.Mock()
job_binary.extra = dict(user='******', password='******')
# bad swift url should raise an exception
job_binary.url = 'notswift://container/object'
self.assertRaises(ex.BadJobBinaryException,
i_s.get_raw_data,
job_binary)
# specifying a container should raise an exception
job_binary.url = 'swift://container'
self.assertRaises(ex.BadJobBinaryException,
i_s.get_raw_data,
job_binary)
# an object that is too large should raise an exception
job_binary.url = 'swift://container/object'
client_instance.head_object = mock.Mock()
header = {'content-length': '2048'}
client_instance.head_object.return_value = header
self.override_config('job_binary_max_KB', 1)
self.assertRaises(ex.DataTooBigException,
i_s.get_raw_data,
job_binary)
client_instance.head_object.assert_called_once_with('container',
'object')
# valid return
client_instance.get_object = mock.Mock()
header = {'content-length': '4'}
body = 'data'
client_instance.head_object.return_value = header
client_instance.get_object.return_value = (header, body)
self.assertEqual(body, i_s.get_raw_data(job_binary))
client_instance.get_object.assert_called_once_with('container',
'object')
def test_get_raw_data(self, swift_client):
client_instance = mock.Mock()
swift_client.return_value = client_instance
job_binary = mock.Mock()
job_binary.extra = dict(user='******', password='******')
# bad swift url should raise an exception
job_binary.url = 'notswift://container/object'
self.assertRaises(ex.BadJobBinaryException, i_s.get_raw_data,
job_binary)
# specifying a container should raise an exception
job_binary.url = 'swift://container'
self.assertRaises(ex.BadJobBinaryException, i_s.get_raw_data,
job_binary)
# an object that is too large should raise an exception
job_binary.url = 'swift://container/object'
client_instance.head_object = mock.Mock()
header = {'content-length': '2048'}
client_instance.head_object.return_value = header
self.override_config('job_binary_max_KB', 1)
self.assertRaises(ex.DataTooBigException, i_s.get_raw_data, job_binary)
client_instance.head_object.assert_called_once_with(
'container', 'object')
# valid return
client_instance.get_object = mock.Mock()
header = {'content-length': '4'}
body = 'data'
client_instance.head_object.return_value = header
client_instance.get_object.return_value = (header, body)
self.assertEqual(body, i_s.get_raw_data(job_binary))
client_instance.get_object.assert_called_once_with(
'container', 'object')
