def _validate_job_binary_url(job_binary_url): if not job_binary_url.startswith(S3_JB_PREFIX): # Sanity check raise ex.BadJobBinaryException( _("URL for binary in S3 must start with %s") % S3_JB_PREFIX) names = _get_names_from_job_binary_url(job_binary_url) if len(names) == 1: # we have a bucket instead of an individual object raise ex.BadJobBinaryException( _("URL for binary in S3 must specify an object not a bucket"))
def _validate_job_binary_url(self, job_binary): if not (job_binary.url.startswith(su.SWIFT_INTERNAL_PREFIX)): # This should have been guaranteed already, # but we'll check just in case. raise ex.BadJobBinaryException( _("Url for binary in internal swift must start with %s") % su.SWIFT_INTERNAL_PREFIX) names = self._get_names_from_url(job_binary.url) if len(names) == 1: # a container has been requested, this is currently unsupported raise ex.BadJobBinaryException( _('Url for binary in internal swift must specify an ' 'object not a container'))
def wrapper(job_binary, *args, **kwargs): if not (job_binary.url.startswith(su.SWIFT_INTERNAL_PREFIX)): # This should have been guaranteed already, # but we'll check just in case. raise ex.BadJobBinaryException( _("Url for binary in internal swift must start with %s") % su.SWIFT_INTERNAL_PREFIX) names = _get_names_from_url(job_binary.url) if len(names) == 1: # a container has been requested, this is currently unsupported raise ex.BadJobBinaryException( _('Url for binary in internal swift must specify an object ' 'not a container')) return f(job_binary, *args, **kwargs)
def validate(self, data, **kwargs): if not kwargs.get('job_binary_id', None): extra = data.get("extra", {}) # Should not be checked during job binary update if (not extra.get("user") or not extra.get("password")) and ( not CONF.use_domain_for_proxy_users): raise ex.BadJobBinaryException()
def get_raw_data(job_binary, proxy_configs=None): conn_kwargs = {} if proxy_configs: conn_kwargs.update(username=proxy_configs.get('proxy_username'), password=proxy_configs.get('proxy_password'), trust_id=proxy_configs.get('proxy_trust_id')) else: conn_kwargs.update(username=job_binary.extra.get('user'), password=job_binary.extra.get('password')) conn = sw.client(**conn_kwargs) if not (job_binary.url.startswith(su.SWIFT_INTERNAL_PREFIX)): # This should have been guaranteed already, # but we'll check just in case. raise ex.BadJobBinaryException( _("Url for binary in internal swift must start with %s") % su.SWIFT_INTERNAL_PREFIX) names = job_binary.url[job_binary.url.index("://") + 3:].split("/", 1) if len(names) == 1: # a container has been requested, this is currently unsupported raise ex.BadJobBinaryException( _('Url for binary in internal swift must specify an object not ' 'a container')) else: container, obj = names # if container name has '.sahara' suffix we need to strip it container = _strip_sahara_suffix(container) try: # First check the size headers = conn.head_object(container, obj) total_KB = int(headers.get('content-length', 0)) / 1024.0 if total_KB > CONF.job_binary_max_KB: raise ex.DataTooBigException( round(total_KB, 1), CONF.job_binary_max_KB, _("Size of swift object (%(size)sKB) is greater " "than maximum (%(maximum)sKB)")) headers, body = conn.get_object(container, obj) except swiftclient.ClientException as e: raise ex.SwiftClientException(six.text_type(e)) return body
def check_job_binary(data, **kwargs): job_binary_location_type = data["url"] extra = data.get("extra", {}) if job_binary_location_type.startswith(su.SWIFT_INTERNAL_PREFIX): if (not extra.get("user") or not extra.get("password")) and ( not CONF.use_domain_for_proxy_users): raise e.BadJobBinaryException() if job_binary_location_type.startswith("internal-db"): internal_uid = job_binary_location_type[len("internal-db://"):] b.check_job_binary_internal_exists(internal_uid)
def check_job_binary(data, **kwargs): job_binary_location_type = data["url"] extra = data.get("extra", {}) # TODO(mattf): remove support for OLD_SWIFT_INTERNAL_PREFIX if job_binary_location_type.startswith(su.SWIFT_INTERNAL_PREFIX) or ( job_binary_location_type.startswith(su.OLD_SWIFT_INTERNAL_PREFIX)): if not extra.get("user") or not extra.get("password"): raise e.BadJobBinaryException() if job_binary_location_type.startswith("internal-db"): internal_uid = job_binary_location_type[len("internal-db://"):] b.check_job_binary_internal_exists(internal_uid)
def check_job_binary(data, **kwargs): job_binary_url = data.get("url", None) extra = data.get("extra", {}) if job_binary_url: if job_binary_url.startswith("internal-db"): internal_uid = job_binary_url.replace( "internal-db://", '') b.check_job_binary_internal_exists(internal_uid) if job_binary_url.startswith(su.SWIFT_INTERNAL_PREFIX): if not kwargs.get('job_binary_id', None): # Should not be checked during job binary update if (not extra.get("user") or not extra.get("password")) and ( not CONF.use_domain_for_proxy_users): raise e.BadJobBinaryException()
def get_raw_data(context, job_binary, proxy_configs=None): if proxy_configs: conn = _get_conn_for_proxy_user(proxy_configs) else: user = job_binary.extra["user"] password = job_binary.extra["password"] conn = _get_conn(user, password) if not (job_binary.url.startswith(su.SWIFT_INTERNAL_PREFIX)): # This should have been guaranteed already, # but we'll check just in case. raise ex.BadJobBinaryException( _("Url for binary in internal swift must start with %s") % su.SWIFT_INTERNAL_PREFIX) names = job_binary.url[job_binary.url.index("://") + 3:].split("/", 1) if len(names) == 1: # We are getting a whole container, return as a dictionary. container = names[0] # if container name has '.sahara' suffix we need to strip it container = _strip_sahara_suffix(container) # First check the size... try: headers = conn.head_container(container) total_KB = int(headers.get('x-container-bytes-used', 0)) / 1024.0 if total_KB > CONF.job_binary_max_KB: raise ex.DataTooBigException( round(total_KB, 1), CONF.job_binary_max_KB, _("Size of swift container (%(size)sKB) is greater " "than maximum (%(maximum)sKB)")) body = {} headers, objects = conn.get_container(container) for item in objects: headers, obj = conn.get_object(container, item["name"]) body[item["name"]] = obj except swiftclient.ClientException as e: raise ex.SwiftClientException(six.text_type(e)) else: container, obj = names # if container name has '.sahara' suffix we need to strip it container = _strip_sahara_suffix(container) try: # First check the size headers = conn.head_object(container, obj) total_KB = int(headers.get('content-length', 0)) / 1024.0 if total_KB > CONF.job_binary_max_KB: raise ex.DataTooBigException( round(total_KB, 1), CONF.job_binary_max_KB, _("Size of swift object (%(size)sKB) is greater " "than maximum (%(maximum)sKB)")) headers, body = conn.get_object(container, obj) except swiftclient.ClientException as e: raise ex.SwiftClientException(six.text_type(e)) return body