def _extract_netloc(self, base_url):
parsed_uri = lib_urlparse(base_url)
short_url = '%(scheme)s://%(netloc)s' % {
'scheme': parsed_uri.scheme,
'netloc': parsed_uri.netloc
}
return short_url
def __init__(self, url,
fs_defaultfs,
logical_name=None,
hdfs_superuser=None,
security_enabled=False,
ssl_cert_ca_verify=True,
temp_dir="/tmp",
umask=0o1022,
hdfs_supergroup=None,
auth_provider=None):
self._url = url
self._superuser = hdfs_superuser
self._security_enabled = security_enabled
self._ssl_cert_ca_verify = ssl_cert_ca_verify
self._temp_dir = temp_dir
self._umask = umask
self._fs_defaultfs = fs_defaultfs
self._logical_name = logical_name
self._supergroup = hdfs_supergroup
self._auth_provider = auth_provider
split = lib_urlparse(fs_defaultfs)
self._scheme = split.scheme
self._netloc = split.netloc
self._is_remote = True
self._has_trash_support = False
self._filebrowser_action = PERMISSION_ACTION_ABFS
self._client = http_client.HttpClient(url, exc_class=WebHdfsException, logger=LOG)
self._root = resource.Resource(self._client)
# To store user info
self._thread_local = threading.local()
LOG.debug("Initializing ABFS : %s (security: %s, superuser: %s)" % (self._url, self._security_enabled, self._superuser))
def __init__(self, raz_url, auth_handler):
self.raz_url = raz_url
self.auth_handler = auth_handler
self.init_time = datetime.now()
self.raz_token = None
o = lib_urlparse(self.raz_url)
self.raz_hostname, self.raz_port = o.netloc.split(':')
self.scheme = o.scheme
def _get_scheme(self, path):
scheme = None
if path:
split = lib_urlparse(path)
scheme = split.scheme if split.scheme else None
ret_scheme = scheme or self._default_scheme
if not ret_scheme:
raise IOError('Can not figure out scheme for path "%s"' % path)
return ret_scheme
def _get_fs_pair(self, src, dst):
"""
Returns two FS for source and destination paths respectively.
If `dst` is not self-contained path assumes it's relative path to `src`.
"""
src_fs = self._get_fs(src)
dst_scheme = lib_urlparse(dst).scheme
if not dst_scheme:
return src_fs, src_fs
return src_fs, self._get_fs(dst)
def __init__(self, raz_url, auth_handler):
self.raz_url = raz_url
self.auth_handler = auth_handler
self.init_time = datetime.now()
self.raz_token = None
o = lib_urlparse(self.raz_url)
if not o.netloc:
raise PopupException(
'Could not parse the host of the Raz server %s' % self.raz_url)
self.raz_hostname, self.raz_port = o.netloc.split(':')
self.scheme = o.scheme
def splitpath(path):
split = lib_urlparse(path)
if split.scheme and split.netloc:
parts = [split.scheme + '://', split.netloc] + split.path.split('/')
elif split.scheme and split.path:
parts = [split.scheme + ':/'] + split.path.split('/')
elif split.scheme:
parts = [split.scheme + ("://" if path.find("://") >= 0 else ":/")]
else:
parts = ['/'] + posixpath.normpath(path).split('/')
# Filter empty parts out
return list(filter(len, parts))
def __init__(self,
url,
fs_defaultfs,
logical_name=None,
hdfs_superuser=None,
security_enabled=False,
ssl_cert_ca_verify=True,
temp_dir="/tmp",
umask=0o1022,
hdfs_supergroup=None,
access_token=None,
token_type=None,
expiration=None,
username=None):
self._url = url
self._superuser = hdfs_superuser
self._security_enabled = security_enabled
self._ssl_cert_ca_verify = ssl_cert_ca_verify
self._temp_dir = temp_dir
self._umask = umask
self.is_sentry_managed = lambda path: False
self._fs_defaultfs = fs_defaultfs
self._logical_name = logical_name
self._supergroup = hdfs_supergroup
self._access_token = access_token
self._token_type = token_type
split = lib_urlparse(fs_defaultfs)
self._scheme = split.scheme
self._netloc = split.netloc
self._is_remote = True
self._has_trash_support = False
self._filebrowser_action = PERMISSION_ACTION_ABFS
self.expiration = expiration
self._user = username
# To store user info
self._thread_local = threading.local() # Unused
self._root = self.get_client(url)
LOG.debug("Initializing ABFS : %s (security: %s, superuser: %s)" %
(self._url, self._security_enabled, self._superuser))
def make_request(self, method, bucket='', key='', headers=None, data='',
query_args=None, sender=None, override_num_retries=None,
retry_handler=None):
if isinstance(bucket, self.bucket_class):
bucket = bucket.name
if isinstance(key, Key):
key = key.name
path = self.calling_format.build_path_base(bucket, key)
boto.log.debug('path=%s' % path)
auth_path = self.calling_format.build_auth_path(bucket, key)
boto.log.debug('auth_path=%s' % auth_path)
# host = self.calling_format.build_host(self.server_name(), bucket)
host = self.calling_format.build_host(self.server_name(), '') # As using signed Url we keep the same hostname as there
if query_args:
path += '?' + query_args
boto.log.debug('path=%s' % path)
auth_path += '?' + query_args
boto.log.debug('auth_path=%s' % auth_path)
params = {}
http_request = self.build_base_http_request(method, path, auth_path,
params, headers, data, host)
# Actual override starts here
LOG.debug('Overriding: %s, %s, %s, %s, %s, %s, %s' % (method, path, auth_path, params, headers, data, host))
signed_url = self.get_signed_url(action='GET', bucket_name=bucket, object_name=key)
LOG.debug(signed_url)
parsed_url = lib_urlparse(signed_url)
# We override instead of re-creating an HTTPRequest
http_request.path = parsed_url.path
if parsed_url.query:
http_request.path += '?' + parsed_url.query
LOG.debug('Overriden: %s' % http_request)
return self._mexe(http_request, sender, override_num_retries,
retry_handler=retry_handler)
def isroot(path):
parsed = lib_urlparse(path)
return (parsed.path == '/'
or parsed.path == '') and parsed.netloc == ''
def check_access(self, method, url, params=None, headers=None):
LOG.debug(
"Check access: method {%s}, url {%s}, params {%s}, headers {%s}" %
(method, url, params, headers))
path = lib_urlparse(url)
url_params = dict([
p.split('=') if '=' in p else (p, '')
for p in path.query.split('&') if path.query
]) # ?delete, ?prefix=/hue
params = params if params is not None else {}
headers = headers if headers is not None else {}
allparams = [
raz_signer.StringListStringMapProto(key=key, value=[val])
for key, val in url_params.items()
]
allparams.extend([
raz_signer.StringListStringMapProto(key=key, value=[val])
for key, val in params.items()
])
headers = [
raz_signer.StringStringMapProto(key=key, value=val)
for key, val in headers.items()
]
endpoint = "%s://%s" % (path.scheme, path.netloc)
resource_path = path.path.lstrip("/")
LOG.debug(
"Preparing sign request with http_method: {%s}, headers: {%s}, parameters: {%s}, endpoint: {%s}, resource_path: {%s}"
% (method, headers, allparams, endpoint, resource_path))
raz_req = raz_signer.SignRequestProto(
endpoint_prefix=self.service_params['endpoint_prefix'],
service_name=self.service_params['service_name'],
endpoint=endpoint,
http_method=method,
headers=headers,
parameters=allparams,
resource_path=resource_path,
time_offset=0)
raz_req_serialized = raz_req.SerializeToString()
signed_request = base64.b64encode(raz_req_serialized)
request_data = {
"requestId": self.requestid,
"serviceType": self.service_params['serviceType'],
"serviceName": self.service_name,
"user": self.username,
"userGroups": [],
"accessTime": "",
"clientIpAddress": "",
"clientType": "",
"clusterName": self.cluster_name,
"clusterType": "",
"sessionId": "",
"context": {
"S3_SIGN_REQUEST": signed_request
}
}
headers = {
"Content-Type": "application/json",
"Accept-Encoding": "gzip,deflate"
}
raz_url = "%s/api/authz/s3/access?delegation=%s" % (self.raz_url,
self.raz_token)
LOG.debug('Raz url: %s' % raz_url)
LOG.debug("Sending access check headers: {%s} request_data: {%s}" %
(headers, request_data))
raz_req = requests.post(raz_url,
headers=headers,
json=request_data,
verify=False)
signed_response_result = None
signed_response = None
if raz_req.ok:
result = raz_req.json().get(
"operResult", False) and raz_req.json()["operResult"]["result"]
if result == "NOT_DETERMINED":
msg = "Failure %s" % raz_req.json()
LOG.error(msg)
raise PopupException(msg)
if result != "ALLOWED":
msg = "Permission missing %s" % raz_req.json()
raise PopupException(msg, error_code=401)
if result == "ALLOWED":
LOG.debug('Received allowed response %s' % raz_req.json())
signed_response_data = raz_req.json(
)["operResult"]["additionalInfo"]
if self.service == 'adls':
LOG.debug("Received SAS %s" %
signed_response_data["ADLS_DSAS"])
return {'token': signed_response_data["ADLS_DSAS"]}
else:
signed_response_result = signed_response_data[
"S3_SIGN_RESPONSE"]
if signed_response_result:
raz_response_proto = raz_signer.SignResponseProto()
signed_response = raz_response_proto.FromString(
base64.b64decode(signed_response_result))
LOG.debug("Received signed Response %s" %
signed_response)
# Signed headers "only"
if signed_response:
return dict([
(i.key, i.value)
for i in signed_response.signer_generated_headers
])
def check_access(self, method, url, params=None, headers=None):
LOG.debug(
"Check access: method {%s}, url {%s}, params {%s}, headers {%s}" %
(method, url, params, headers))
path = lib_urlparse(url)
url_params = dict([
p.split('=') if '=' in p else (p, '')
for p in path.query.split('&') if path.query
]) # ?delete, ?prefix=/hue
params = params if params is not None else {}
headers = headers if headers is not None else {}
endpoint = "%s://%s" % (path.scheme, path.netloc)
resource_path = path.path.lstrip("/")
request_data = {
"requestId": self.requestid,
"serviceType": self.service_params['serviceType'],
"serviceName": self.service_name,
"user": self.username,
"userGroups": [],
"clientIpAddress": "",
"clientType": "",
"clusterName": self.cluster_name,
"clusterType": "",
"sessionId": "",
"accessTime": "",
"context": {}
}
request_headers = {"Content-Type": "application/json"}
raz_url = "%s/api/authz/%s/access?delegation=%s" % (
self.raz_url, self.service, self.raz_token)
if self.service == 'adls':
self._make_adls_request(request_data, path, resource_path)
elif self.service == 's3':
self._make_s3_request(request_data, request_headers, method,
params, headers, url_params, endpoint,
resource_path)
LOG.debug('Raz url: %s' % raz_url)
LOG.debug("Sending access check headers: {%s} request_data: {%s}" %
(request_headers, request_data))
raz_req = requests.post(raz_url,
headers=request_headers,
json=request_data,
verify=False)
signed_response_result = None
signed_response = None
if raz_req.ok:
result = raz_req.json().get(
"operResult", False) and raz_req.json()["operResult"]["result"]
if result == "NOT_DETERMINED":
msg = "Failure %s" % raz_req.json()
LOG.error(msg)
raise PopupException(msg)
if result != "ALLOWED":
msg = "Permission missing %s" % raz_req.json()
raise PopupException(msg, error_code=401)
if result == "ALLOWED":
LOG.debug('Received allowed response %s' % raz_req.json())
signed_response_data = raz_req.json(
)["operResult"]["additionalInfo"]
if self.service == 'adls':
LOG.debug("Received SAS %s" %
signed_response_data["ADLS_DSAS"])
return {'token': signed_response_data["ADLS_DSAS"]}
else:
signed_response_result = signed_response_data[
"S3_SIGN_RESPONSE"]
if signed_response_result is not None:
raz_response_proto = raz_signer.SignResponseProto()
signed_response = raz_response_proto.FromString(
base64.b64decode(signed_response_result))
LOG.debug("Received signed Response %s" %
signed_response)
# Signed headers "only"
if signed_response is not None:
return dict([
(i.key, i.value)
for i in signed_response.signer_generated_headers
])
