def _validate_job_binary_url(job_binary_url):
if not job_binary_url.startswith(S3_JB_PREFIX):
# Sanity check
raise ex.BadJobBinaryException(
_("URL for binary in S3 must start with %s") % S3_JB_PREFIX)
names = _get_names_from_job_binary_url(job_binary_url)
if len(names) == 1:
# we have a bucket instead of an individual object
raise ex.BadJobBinaryException(
_("URL for binary in S3 must specify an object not a bucket"))
def _validate_job_binary_url(self, job_binary):
if not (job_binary.url.startswith(su.SWIFT_INTERNAL_PREFIX)):
# This should have been guaranteed already,
# but we'll check just in case.
raise ex.BadJobBinaryException(
_("Url for binary in internal swift must start with %s")
% su.SWIFT_INTERNAL_PREFIX)
names = self._get_names_from_url(job_binary.url)
if len(names) == 1:
# a container has been requested, this is currently unsupported
raise ex.BadJobBinaryException(
_('Url for binary in internal swift must specify an '
'object not a container'))
def wrapper(job_binary, *args, **kwargs):
if not (job_binary.url.startswith(su.SWIFT_INTERNAL_PREFIX)):
# This should have been guaranteed already,
# but we'll check just in case.
raise ex.BadJobBinaryException(
_("Url for binary in internal swift must start with %s") %
su.SWIFT_INTERNAL_PREFIX)
names = _get_names_from_url(job_binary.url)
if len(names) == 1:
# a container has been requested, this is currently unsupported
raise ex.BadJobBinaryException(
_('Url for binary in internal swift must specify an object '
'not a container'))
return f(job_binary, *args, **kwargs)
def validate(self, data, **kwargs):
if not kwargs.get('job_binary_id', None):
extra = data.get("extra", {})
# Should not be checked during job binary update
if (not extra.get("user") or not extra.get("password")) and (
not CONF.use_domain_for_proxy_users):
raise ex.BadJobBinaryException()
def get_raw_data(job_binary, proxy_configs=None):
conn_kwargs = {}
if proxy_configs:
conn_kwargs.update(username=proxy_configs.get('proxy_username'),
password=proxy_configs.get('proxy_password'),
trust_id=proxy_configs.get('proxy_trust_id'))
else:
conn_kwargs.update(username=job_binary.extra.get('user'),
password=job_binary.extra.get('password'))
conn = sw.client(**conn_kwargs)
if not (job_binary.url.startswith(su.SWIFT_INTERNAL_PREFIX)):
# This should have been guaranteed already,
# but we'll check just in case.
raise ex.BadJobBinaryException(
_("Url for binary in internal swift must start with %s") %
su.SWIFT_INTERNAL_PREFIX)
names = job_binary.url[job_binary.url.index("://") + 3:].split("/", 1)
if len(names) == 1:
# a container has been requested, this is currently unsupported
raise ex.BadJobBinaryException(
_('Url for binary in internal swift must specify an object not '
'a container'))
else:
container, obj = names
# if container name has '.sahara' suffix we need to strip it
container = _strip_sahara_suffix(container)
try:
# First check the size
headers = conn.head_object(container, obj)
total_KB = int(headers.get('content-length', 0)) / 1024.0
if total_KB > CONF.job_binary_max_KB:
raise ex.DataTooBigException(
round(total_KB, 1), CONF.job_binary_max_KB,
_("Size of swift object (%(size)sKB) is greater "
"than maximum (%(maximum)sKB)"))
headers, body = conn.get_object(container, obj)
except swiftclient.ClientException as e:
raise ex.SwiftClientException(six.text_type(e))
return body
def check_job_binary(data, **kwargs):
job_binary_location_type = data["url"]
extra = data.get("extra", {})
if job_binary_location_type.startswith(su.SWIFT_INTERNAL_PREFIX):
if (not extra.get("user") or not extra.get("password")) and (
not CONF.use_domain_for_proxy_users):
raise e.BadJobBinaryException()
if job_binary_location_type.startswith("internal-db"):
internal_uid = job_binary_location_type[len("internal-db://"):]
b.check_job_binary_internal_exists(internal_uid)
def check_job_binary(data, **kwargs):
job_binary_location_type = data["url"]
extra = data.get("extra", {})
# TODO(mattf): remove support for OLD_SWIFT_INTERNAL_PREFIX
if job_binary_location_type.startswith(su.SWIFT_INTERNAL_PREFIX) or (
job_binary_location_type.startswith(su.OLD_SWIFT_INTERNAL_PREFIX)):
if not extra.get("user") or not extra.get("password"):
raise e.BadJobBinaryException()
if job_binary_location_type.startswith("internal-db"):
internal_uid = job_binary_location_type[len("internal-db://"):]
b.check_job_binary_internal_exists(internal_uid)
def check_job_binary(data, **kwargs):
job_binary_url = data.get("url", None)
extra = data.get("extra", {})
if job_binary_url:
if job_binary_url.startswith("internal-db"):
internal_uid = job_binary_url.replace(
"internal-db://", '')
b.check_job_binary_internal_exists(internal_uid)
if job_binary_url.startswith(su.SWIFT_INTERNAL_PREFIX):
if not kwargs.get('job_binary_id', None):
# Should not be checked during job binary update
if (not extra.get("user") or not extra.get("password")) and (
not CONF.use_domain_for_proxy_users):
raise e.BadJobBinaryException()
def get_raw_data(context, job_binary, proxy_configs=None):
if proxy_configs:
conn = _get_conn_for_proxy_user(proxy_configs)
else:
user = job_binary.extra["user"]
password = job_binary.extra["password"]
conn = _get_conn(user, password)
if not (job_binary.url.startswith(su.SWIFT_INTERNAL_PREFIX)):
# This should have been guaranteed already,
# but we'll check just in case.
raise ex.BadJobBinaryException(
_("Url for binary in internal swift must start with %s")
% su.SWIFT_INTERNAL_PREFIX)
names = job_binary.url[job_binary.url.index("://") + 3:].split("/", 1)
if len(names) == 1:
# We are getting a whole container, return as a dictionary.
container = names[0]
# if container name has '.sahara' suffix we need to strip it
container = _strip_sahara_suffix(container)
# First check the size...
try:
headers = conn.head_container(container)
total_KB = int(headers.get('x-container-bytes-used', 0)) / 1024.0
if total_KB > CONF.job_binary_max_KB:
raise ex.DataTooBigException(
round(total_KB, 1), CONF.job_binary_max_KB,
_("Size of swift container (%(size)sKB) is greater "
"than maximum (%(maximum)sKB)"))
body = {}
headers, objects = conn.get_container(container)
for item in objects:
headers, obj = conn.get_object(container, item["name"])
body[item["name"]] = obj
except swiftclient.ClientException as e:
raise ex.SwiftClientException(six.text_type(e))
else:
container, obj = names
# if container name has '.sahara' suffix we need to strip it
container = _strip_sahara_suffix(container)
try:
# First check the size
headers = conn.head_object(container, obj)
total_KB = int(headers.get('content-length', 0)) / 1024.0
if total_KB > CONF.job_binary_max_KB:
raise ex.DataTooBigException(
round(total_KB, 1), CONF.job_binary_max_KB,
_("Size of swift object (%(size)sKB) is greater "
"than maximum (%(maximum)sKB)"))
headers, body = conn.get_object(container, obj)
except swiftclient.ClientException as e:
raise ex.SwiftClientException(six.text_type(e))
return body