def _extract_netloc(self, base_url): parsed_uri = lib_urlparse(base_url) short_url = '%(scheme)s://%(netloc)s' % { 'scheme': parsed_uri.scheme, 'netloc': parsed_uri.netloc } return short_url
def __init__(self, url, fs_defaultfs, logical_name=None, hdfs_superuser=None, security_enabled=False, ssl_cert_ca_verify=True, temp_dir="/tmp", umask=0o1022, hdfs_supergroup=None, auth_provider=None): self._url = url self._superuser = hdfs_superuser self._security_enabled = security_enabled self._ssl_cert_ca_verify = ssl_cert_ca_verify self._temp_dir = temp_dir self._umask = umask self._fs_defaultfs = fs_defaultfs self._logical_name = logical_name self._supergroup = hdfs_supergroup self._auth_provider = auth_provider split = lib_urlparse(fs_defaultfs) self._scheme = split.scheme self._netloc = split.netloc self._is_remote = True self._has_trash_support = False self._filebrowser_action = PERMISSION_ACTION_ABFS self._client = http_client.HttpClient(url, exc_class=WebHdfsException, logger=LOG) self._root = resource.Resource(self._client) # To store user info self._thread_local = threading.local() LOG.debug("Initializing ABFS : %s (security: %s, superuser: %s)" % (self._url, self._security_enabled, self._superuser))
def __init__(self, raz_url, auth_handler): self.raz_url = raz_url self.auth_handler = auth_handler self.init_time = datetime.now() self.raz_token = None o = lib_urlparse(self.raz_url) self.raz_hostname, self.raz_port = o.netloc.split(':') self.scheme = o.scheme
def _get_scheme(self, path): scheme = None if path: split = lib_urlparse(path) scheme = split.scheme if split.scheme else None ret_scheme = scheme or self._default_scheme if not ret_scheme: raise IOError('Can not figure out scheme for path "%s"' % path) return ret_scheme
def _get_fs_pair(self, src, dst): """ Returns two FS for source and destination paths respectively. If `dst` is not self-contained path assumes it's relative path to `src`. """ src_fs = self._get_fs(src) dst_scheme = lib_urlparse(dst).scheme if not dst_scheme: return src_fs, src_fs return src_fs, self._get_fs(dst)
def __init__(self, raz_url, auth_handler): self.raz_url = raz_url self.auth_handler = auth_handler self.init_time = datetime.now() self.raz_token = None o = lib_urlparse(self.raz_url) if not o.netloc: raise PopupException( 'Could not parse the host of the Raz server %s' % self.raz_url) self.raz_hostname, self.raz_port = o.netloc.split(':') self.scheme = o.scheme
def splitpath(path): split = lib_urlparse(path) if split.scheme and split.netloc: parts = [split.scheme + '://', split.netloc] + split.path.split('/') elif split.scheme and split.path: parts = [split.scheme + ':/'] + split.path.split('/') elif split.scheme: parts = [split.scheme + ("://" if path.find("://") >= 0 else ":/")] else: parts = ['/'] + posixpath.normpath(path).split('/') # Filter empty parts out return list(filter(len, parts))
def __init__(self, url, fs_defaultfs, logical_name=None, hdfs_superuser=None, security_enabled=False, ssl_cert_ca_verify=True, temp_dir="/tmp", umask=0o1022, hdfs_supergroup=None, access_token=None, token_type=None, expiration=None, username=None): self._url = url self._superuser = hdfs_superuser self._security_enabled = security_enabled self._ssl_cert_ca_verify = ssl_cert_ca_verify self._temp_dir = temp_dir self._umask = umask self.is_sentry_managed = lambda path: False self._fs_defaultfs = fs_defaultfs self._logical_name = logical_name self._supergroup = hdfs_supergroup self._access_token = access_token self._token_type = token_type split = lib_urlparse(fs_defaultfs) self._scheme = split.scheme self._netloc = split.netloc self._is_remote = True self._has_trash_support = False self._filebrowser_action = PERMISSION_ACTION_ABFS self.expiration = expiration self._user = username # To store user info self._thread_local = threading.local() # Unused self._root = self.get_client(url) LOG.debug("Initializing ABFS : %s (security: %s, superuser: %s)" % (self._url, self._security_enabled, self._superuser))
def make_request(self, method, bucket='', key='', headers=None, data='', query_args=None, sender=None, override_num_retries=None, retry_handler=None): if isinstance(bucket, self.bucket_class): bucket = bucket.name if isinstance(key, Key): key = key.name path = self.calling_format.build_path_base(bucket, key) boto.log.debug('path=%s' % path) auth_path = self.calling_format.build_auth_path(bucket, key) boto.log.debug('auth_path=%s' % auth_path) # host = self.calling_format.build_host(self.server_name(), bucket) host = self.calling_format.build_host(self.server_name(), '') # As using signed Url we keep the same hostname as there if query_args: path += '?' + query_args boto.log.debug('path=%s' % path) auth_path += '?' + query_args boto.log.debug('auth_path=%s' % auth_path) params = {} http_request = self.build_base_http_request(method, path, auth_path, params, headers, data, host) # Actual override starts here LOG.debug('Overriding: %s, %s, %s, %s, %s, %s, %s' % (method, path, auth_path, params, headers, data, host)) signed_url = self.get_signed_url(action='GET', bucket_name=bucket, object_name=key) LOG.debug(signed_url) parsed_url = lib_urlparse(signed_url) # We override instead of re-creating an HTTPRequest http_request.path = parsed_url.path if parsed_url.query: http_request.path += '?' + parsed_url.query LOG.debug('Overriden: %s' % http_request) return self._mexe(http_request, sender, override_num_retries, retry_handler=retry_handler)
def isroot(path): parsed = lib_urlparse(path) return (parsed.path == '/' or parsed.path == '') and parsed.netloc == ''
def check_access(self, method, url, params=None, headers=None): LOG.debug( "Check access: method {%s}, url {%s}, params {%s}, headers {%s}" % (method, url, params, headers)) path = lib_urlparse(url) url_params = dict([ p.split('=') if '=' in p else (p, '') for p in path.query.split('&') if path.query ]) # ?delete, ?prefix=/hue params = params if params is not None else {} headers = headers if headers is not None else {} allparams = [ raz_signer.StringListStringMapProto(key=key, value=[val]) for key, val in url_params.items() ] allparams.extend([ raz_signer.StringListStringMapProto(key=key, value=[val]) for key, val in params.items() ]) headers = [ raz_signer.StringStringMapProto(key=key, value=val) for key, val in headers.items() ] endpoint = "%s://%s" % (path.scheme, path.netloc) resource_path = path.path.lstrip("/") LOG.debug( "Preparing sign request with http_method: {%s}, headers: {%s}, parameters: {%s}, endpoint: {%s}, resource_path: {%s}" % (method, headers, allparams, endpoint, resource_path)) raz_req = raz_signer.SignRequestProto( endpoint_prefix=self.service_params['endpoint_prefix'], service_name=self.service_params['service_name'], endpoint=endpoint, http_method=method, headers=headers, parameters=allparams, resource_path=resource_path, time_offset=0) raz_req_serialized = raz_req.SerializeToString() signed_request = base64.b64encode(raz_req_serialized) request_data = { "requestId": self.requestid, "serviceType": self.service_params['serviceType'], "serviceName": self.service_name, "user": self.username, "userGroups": [], "accessTime": "", "clientIpAddress": "", "clientType": "", "clusterName": self.cluster_name, "clusterType": "", "sessionId": "", "context": { "S3_SIGN_REQUEST": signed_request } } headers = { "Content-Type": "application/json", "Accept-Encoding": "gzip,deflate" } raz_url = "%s/api/authz/s3/access?delegation=%s" % (self.raz_url, self.raz_token) LOG.debug('Raz url: %s' % raz_url) LOG.debug("Sending access check headers: {%s} request_data: {%s}" % (headers, request_data)) raz_req = requests.post(raz_url, headers=headers, json=request_data, verify=False) signed_response_result = None signed_response = None if raz_req.ok: result = raz_req.json().get( "operResult", False) and raz_req.json()["operResult"]["result"] if result == "NOT_DETERMINED": msg = "Failure %s" % raz_req.json() LOG.error(msg) raise PopupException(msg) if result != "ALLOWED": msg = "Permission missing %s" % raz_req.json() raise PopupException(msg, error_code=401) if result == "ALLOWED": LOG.debug('Received allowed response %s' % raz_req.json()) signed_response_data = raz_req.json( )["operResult"]["additionalInfo"] if self.service == 'adls': LOG.debug("Received SAS %s" % signed_response_data["ADLS_DSAS"]) return {'token': signed_response_data["ADLS_DSAS"]} else: signed_response_result = signed_response_data[ "S3_SIGN_RESPONSE"] if signed_response_result: raz_response_proto = raz_signer.SignResponseProto() signed_response = raz_response_proto.FromString( base64.b64decode(signed_response_result)) LOG.debug("Received signed Response %s" % signed_response) # Signed headers "only" if signed_response: return dict([ (i.key, i.value) for i in signed_response.signer_generated_headers ])
def check_access(self, method, url, params=None, headers=None): LOG.debug( "Check access: method {%s}, url {%s}, params {%s}, headers {%s}" % (method, url, params, headers)) path = lib_urlparse(url) url_params = dict([ p.split('=') if '=' in p else (p, '') for p in path.query.split('&') if path.query ]) # ?delete, ?prefix=/hue params = params if params is not None else {} headers = headers if headers is not None else {} endpoint = "%s://%s" % (path.scheme, path.netloc) resource_path = path.path.lstrip("/") request_data = { "requestId": self.requestid, "serviceType": self.service_params['serviceType'], "serviceName": self.service_name, "user": self.username, "userGroups": [], "clientIpAddress": "", "clientType": "", "clusterName": self.cluster_name, "clusterType": "", "sessionId": "", "accessTime": "", "context": {} } request_headers = {"Content-Type": "application/json"} raz_url = "%s/api/authz/%s/access?delegation=%s" % ( self.raz_url, self.service, self.raz_token) if self.service == 'adls': self._make_adls_request(request_data, path, resource_path) elif self.service == 's3': self._make_s3_request(request_data, request_headers, method, params, headers, url_params, endpoint, resource_path) LOG.debug('Raz url: %s' % raz_url) LOG.debug("Sending access check headers: {%s} request_data: {%s}" % (request_headers, request_data)) raz_req = requests.post(raz_url, headers=request_headers, json=request_data, verify=False) signed_response_result = None signed_response = None if raz_req.ok: result = raz_req.json().get( "operResult", False) and raz_req.json()["operResult"]["result"] if result == "NOT_DETERMINED": msg = "Failure %s" % raz_req.json() LOG.error(msg) raise PopupException(msg) if result != "ALLOWED": msg = "Permission missing %s" % raz_req.json() raise PopupException(msg, error_code=401) if result == "ALLOWED": LOG.debug('Received allowed response %s' % raz_req.json()) signed_response_data = raz_req.json( )["operResult"]["additionalInfo"] if self.service == 'adls': LOG.debug("Received SAS %s" % signed_response_data["ADLS_DSAS"]) return {'token': signed_response_data["ADLS_DSAS"]} else: signed_response_result = signed_response_data[ "S3_SIGN_RESPONSE"] if signed_response_result is not None: raz_response_proto = raz_signer.SignResponseProto() signed_response = raz_response_proto.FromString( base64.b64decode(signed_response_result)) LOG.debug("Received signed Response %s" % signed_response) # Signed headers "only" if signed_response is not None: return dict([ (i.key, i.value) for i in signed_response.signer_generated_headers ])