def get_client(self, url): if RAZ.IS_ENABLED.get(): return resource.Resource(RazHttpClient()) else: return resource.Resource( http_client.HttpClient(url, exc_class=WebHdfsException, logger=LOG))
def __init__(self, url, fs_defaultfs, logical_name=None, hdfs_superuser=None, security_enabled=False, ssl_cert_ca_verify=True, temp_dir="/tmp", umask=01022): self._url = url self._superuser = hdfs_superuser self._security_enabled = security_enabled self._ssl_cert_ca_verify = ssl_cert_ca_verify self._temp_dir = temp_dir self._umask = umask self._fs_defaultfs = fs_defaultfs self._logical_name = logical_name self._client = self._make_client(url, security_enabled, ssl_cert_ca_verify) self._root = resource.Resource(self._client) # To store user info self._thread_local = threading.local() LOG.debug( "Initializing Hadoop WebHdfs: %s (security: %s, superuser: %s)" % (self._url, self._security_enabled, self._superuser))
def __init__(self, url, fs_defaultfs, logical_name=None, hdfs_superuser=None, security_enabled=False, ssl_cert_ca_verify=True, temp_dir="/tmp", umask=0o1022, hdfs_supergroup=None, auth_provider=None): self._url = url self._superuser = hdfs_superuser self._security_enabled = security_enabled self._ssl_cert_ca_verify = ssl_cert_ca_verify self._temp_dir = temp_dir self._umask = umask self._fs_defaultfs = fs_defaultfs self._logical_name = logical_name self._supergroup = hdfs_supergroup self._auth_provider = auth_provider split = urlparse(fs_defaultfs) self._scheme = split.scheme self._netloc = split.netloc self._is_remote = True self._has_trash_support = False self._filebrowser_action = PERMISSION_ACTION_ABFS self._client = http_client.HttpClient(url, exc_class=WebHdfsException, logger=LOG) self._root = resource.Resource(self._client) # To store user info self._thread_local = threading.local() LOG.debug("Initializing ABFS : %s (security: %s, superuser: %s)" % (self._url, self._security_enabled, self._superuser))
def __init__(self, solr_url=None, user=None, security_enabled=False, ssl_cert_ca_verify=SSL_CERT_CA_VERIFY.get()): if solr_url is None: solr_url = SOLR_URL.get() self._url = solr_url self._user = user self._client = HttpClient(self._url, logger=LOG) self.security_enabled = security_enabled or SECURITY_ENABLED.get() if self.security_enabled: self._client.set_kerberos_auth() self._client.set_verify(ssl_cert_ca_verify) self._root = resource.Resource(self._client) # The Kerberos handshake requires two requests in order to authenticate, # but if our first request is a PUT/POST, it might flat-out reject the # first request if the body is too large. So, connect here in order to get # a cookie so future PUT/POSTs will be pre-authenticated. if self.security_enabled: self._root.invoke('HEAD', '/')
def __init__( self, url, fs_defaultfs, logical_name=None, hdfs_superuser=None, security_enabled=False, ssl_cert_ca_verify=True, temp_dir="/tmp", umask=0o1022, hdfs_supergroup=None): self._url = url self._superuser = hdfs_superuser self._security_enabled = security_enabled self._ssl_cert_ca_verify = ssl_cert_ca_verify self._temp_dir = temp_dir self._umask = umask self._fs_defaultfs = fs_defaultfs self._logical_name = logical_name self._supergroup = hdfs_supergroup self._scheme = "" self._netloc = ""; self._is_remote = False self._has_trash_support = True self.expiration = None self._client = self._make_client(url, security_enabled, ssl_cert_ca_verify) self._root = resource.Resource(self._client) # To store user info self._thread_local = threading.local() LOG.debug("Initializing Hadoop WebHdfs: %s (security: %s, superuser: %s)" % (self._url, self._security_enabled, self._superuser))
def _invoke_with_redirect(self, method, path, params=None, data=None, headers=None): """ Issue a request, and expect a redirect, and then submit the data to the redirected location. This is used for create, write, etc. Returns the response from the redirected request. """ next_url = None try: # Do not pass data in the first leg. self._root.invoke(method, path, params, headers=headers) except WebHdfsException as ex: # This is expected. We get a 307 redirect. # The following call may throw. next_url = self._get_redirect_url(ex) if next_url is None: raise WebHdfsException(_("Failed to create '%s'. HDFS did not return a redirect") % path) # Now talk to the real thing. The redirect url already includes the params. client = self._make_client(next_url, self.security_enabled, self.ssl_cert_ca_verify) # Make sure to reuse the session in order to preserve the Kerberos cookies. client._session = self._client._session if headers is None: headers = {} headers["Content-Type"] = 'application/octet-stream' return resource.Resource(client).invoke(method, data=data, headers=headers)
def get_access_token(cls, session_code, **options): remote_url = options.get( 'remote_url', VCS[GITHUB_OFFICIAL].REMOTE_URL.get()).strip('/') client_id = options.get('client_id', VCS[GITHUB_OFFICIAL].CLIENT_ID.get()) client_secret = options.get('client_secret', VCS[GITHUB_OFFICIAL].CLIENT_SECRET.get()) try: client = HttpClient(remote_url, logger=LOG) root = resource.Resource(client) data = { 'client_id': client_id, 'client_secret': client_secret, 'code': session_code } headers = { 'content-type': 'application/json', 'Accept': 'application/json' } response = root.post('login/oauth/access_token', headers=headers, data=json.dumps(data)) result = cls._get_json(response) return result['access_token'] except RestException as e: raise GithubClientException( 'Failed to request access token from GitHub: %s' % e) except KeyError: raise GithubClientException( 'Failed to find access_token in GitHub oAuth response')
def __init__(self, solr_url, user): self._url = solr_url self._user = user self._client = HttpClient(self._url, logger=LOG) self.security_enabled = SECURITY_ENABLED.get() if self.security_enabled: self._client.set_kerberos_auth() self._root = resource.Resource(self._client)
def azure_metadata(): global AZURE_METADATA if AZURE_METADATA is None: from desktop.lib.rest import http_client, resource client = http_client.HttpClient(META_DATA_URL, logger=LOG) root = resource.Resource(client) try: AZURE_METADATA = root.get('/compute', params={'api-version': '2019-06-04', 'format': 'json'}, headers={'Metadata': 'true'}) except Exception as e: AZURE_METADATA = False return AZURE_METADATA
def is_authenticated(cls, access_token, **options): api_url = options.get('api_url', GITHUB_API_URL.get()).strip('/') try: client = HttpClient(api_url, logger=LOG) root = resource.Resource(client) params = (('access_token', access_token), ) root.get('user', params=params) return True except RestException: return False
def __init__(self, api_url=None, user=None, password=None): self._api_url = (api_url or NAVIGATOR.API_URL.get()).strip('/') self._username = user or NAVIGATOR.AUTH_USERNAME.get() self._password = password or NAVIGATOR.AUTH_PASSWORD.get() self._client = HttpClient(self._api_url, logger=LOG) self._client.set_basic_auth(self._username, self._password) self._root = resource.Resource(self._client) self.__headers = {} self.__params = ()
def __init__(self, url=None, aws_access_key_id=None, aws_secret_access_key=None): self._access_key_id = aws_access_key_id self._secret_access_key = aws_secret_access_key self._url = url self._client = http_client.HttpClient(url, logger=LOG) self._root = resource.Resource(self._client) self._token = None
def __init__(self, api_url=None, product_name=None, product_secret=None, ssl_cert_ca_verify=OPTIMIZER.SSL_CERT_CA_VERIFY.get(), product_auth_secret=None): self._api_url = (api_url or OPTIMIZER.API_URL.get()).strip('/') self._product_name = product_name if product_name else OPTIMIZER.PRODUCT_NAME.get() self._product_secret = product_secret if product_secret else OPTIMIZER.PRODUCT_SECRET.get() self._product_auth_secret = product_auth_secret if product_auth_secret else OPTIMIZER.PRODUCT_AUTH_SECRET.get() self._email = OPTIMIZER.EMAIL.get() self._email_password = OPTIMIZER.EMAIL_PASSWORD.get() self._client = HttpClient(self._api_url, logger=LOG) self._client.set_verify(ssl_cert_ca_verify) self._root = resource.Resource(self._client)
def get_client(self, url): if RAZ.IS_ENABLED.get(): client = RazHttpClient(self._user, url, exc_class=WebHdfsException, logger=LOG) else: client = http_client.HttpClient(url, exc_class=WebHdfsException, logger=LOG) return resource.Resource(client)
def __init__(self, user=None): self._api_url = '%s/%s' % (NAVIGATOR.API_URL.get().strip('/'), VERSION) self._username = NAVIGATOR.AUTH_USERNAME.get() self._password = NAVIGATOR.AUTH_PASSWORD.get() self.user = user self._client = HttpClient(self._api_url, logger=LOG) self._client.set_basic_auth(self._username, self._password) self._root = resource.Resource(self._client, urlencode=False) # For search_entities_interactive self.__headers = {} self.__params = ()
def __init__(self, user=None, address=None, dt_path=None, path=None, security=None): self.user = user self.address = address self.dt_path = dt_path self.path = path self.security = security self._client = http_client.HttpClient(self.address, logger=LOG) self._root = resource.Resource(self._client)
def __init__(self, user=None): self._api_url = '%s/%s' % (NAVIGATOR.API_URL.get().strip('/'), VERSION) self._username = get_navigator_auth_username() self._password = get_navigator_auth_password() self.user = user # Navigator does not support Kerberos authentication while other components usually requires it self._client = UnsecureHttpClient(self._api_url, logger=LOG) self._client.set_basic_auth(self._username, self._password) self._root = resource.Resource(self._client, urlencode=False) # For search_entities_interactive self.__headers = {} self.__params = ()
def __init__(self, **options): self._github_base_url = options.get('remote_url', VCS[GITHUB_OFFICIAL].REMOTE_URL.get()).strip('/') self._api_url = options.get('api_url', VCS[GITHUB_OFFICIAL].API_URL.get()).strip('/') self._client = HttpClient(self._api_url, logger=LOG) self._root = resource.Resource(self._client) self.__headers = {} access_token = options.get('access_token') if access_token: self.__headers['Authorization'] = 'token %s' % access_token # TODO: Redact access_token from logs self.__params = ()
def __init__(self, user=None): super(AtlasApi, self).__init__(user) self._api_url = CATALOG.API_URL.get().strip('/') self._username = CATALOG.SERVER_USER.get() self._password = get_catalog_auth_password() # Navigator does not support Kerberos authentication while other components usually requires it self._client = UnsecureHttpClient(self._api_url, logger=LOG) self._client.set_basic_auth(self._username, self._password) self._root = resource.Resource(self._client, urlencode=False) # For search_entities_interactive self.__headers = {} self.__params = ()
def __init__(self, solr_url, user, security_enabled=SECURITY_ENABLED.get()): self._url = solr_url self._user = user self._client = HttpClient(self._url, logger=LOG) self.security_enabled = security_enabled if self.security_enabled: self._client.set_kerberos_auth() self._root = resource.Resource(self._client) # The Kerberos handshake requires two requests in order to authenticate, # but if our first request is a PUT/POST, it might flat-out reject the # first request if the body is too large. So, connect here in order to get # a cookie so future PUT/POSTs will be pre-authenticated. if self.security_enabled: self._root.invoke('HEAD', '/')
def __init__(self, user=None): super(AtlasApi, self).__init__(user) self._api_url = CATALOG.API_URL.get().strip('/') + "/api/atlas" self._username = CATALOG.SERVER_USER.get() self._password = CATALOG.SERVER_PASSWORD.get() self._client = HttpClient(self._api_url, logger=LOG) if CATALOG.KERBEROS_ENABLED.get(): self._client.set_kerberos_auth() elif self._password: self._client.set_basic_auth(self._username, self._password) self._root = resource.Resource( self._client, urlencode=False) # For search_entities_interactive self.__headers = {} self.__params = ()
def __init__(self, url, fs_defaultfs, hdfs_superuser=None, security_enabled=False, temp_dir="/tmp"): self._url = url self._superuser = hdfs_superuser self._security_enabled = security_enabled self._temp_dir = temp_dir self._fs_defaultfs = fs_defaultfs self._client = self._make_client(url, security_enabled) self._root = resource.Resource(self._client) # To store user info self._thread_local = threading.local() LOG.debug("Initializing Hadoop WebHdfs: %s (security: %s, superuser: %s)" % (self._url, self._security_enabled, self._superuser))
def __init__(self, user=None): super(AtlasApi, self).__init__(user) self._api_url = None # Checking which server is active server if there are multiple else use the one thats listed atlas_servers = CATALOG.API_URL.get().replace("%20", "").replace( "['", "").replace("']", "").replace("'", "").split(',') for atlas_server in atlas_servers: atlas_url = atlas_server.strip().strip( '/') + '/api/atlas/admin/status' response = requests.get(atlas_url) atlas_is_active = response.json() if "ACTIVE" in atlas_is_active["Status"]: LOG.debug('Setting Atlas API endpoint to: %s' % atlas_server) self._api_url = atlas_server.strip().strip('/') + "/api/atlas" break if self._api_url is None: self._api_url = CATALOG.API_URL.get() LOG.warning('No Atlas server available for use, defaulting to %s' % self._api_url) self._username = CATALOG.SERVER_USER.get() self._password = CATALOG.SERVER_PASSWORD.get() self._client = HttpClient(self._api_url, logger=LOG) if CATALOG.KERBEROS_ENABLED.get(): self._client.set_kerberos_auth() elif self._password: self._client.set_basic_auth(self._username, self._password) self._root = resource.Resource( self._client, urlencode=False) # For search_entities_interactive self.__headers = {} self.__params = ()
ssl_cert_ca_verify=True, temp_dir="/tmp", umask=01022, hdfs_supergroup=None): self._url = url self._superuser = hdfs_superuser self._security_enabled = security_enabled self._ssl_cert_ca_verify = ssl_cert_ca_verify self._temp_dir = temp_dir self._umask = umask self._fs_defaultfs = fs_defaultfs self._logical_name = logical_name self._supergroup = hdfs_supergroup self._client = self._make_client(url, security_enabled, ssl_cert_ca_verify) self._root = resource.Resource(self._client) # To store user info self._thread_local = threading.local() LOG.debug("Initializing Hadoop WebHdfs: %s (security: %s, superuser: %s)" % (self._url, self._security_enabled, self._superuser)) @classmethod def from_config(cls, hdfs_config): fs_defaultfs = hdfs_config.FS_DEFAULTFS.get() return cls(url=_get_service_url(hdfs_config), fs_defaultfs=fs_defaultfs, logical_name=hdfs_config.LOGICAL_NAME.get(), security_enabled=hdfs_config.SECURITY_ENABLED.get(), ssl_cert_ca_verify=hdfs_config.SSL_CERT_CA_VERIFY.get(),
def get_client(self, url): return resource.Resource( http_client.HttpClient(url, exc_class=WebHdfsException, logger=LOG))
def __init__(self): self._remote_url = VCS[GIT_READ_ONLY].REMOTE_URL.get().strip('/') self._api_url = VCS[GIT_READ_ONLY].API_URL.get().strip('/') self._client = HttpClient(self._api_url, logger=LOG) self._root = resource.Resource(self._client)
try: # Do not pass data in the first leg. self._root.invoke(method, path, params) except WebHdfsException, ex: # This is expected. We get a 307 redirect. # The following call may throw. next_url = self._get_redirect_url(ex) if next_url is None: raise WebHdfsException( "Failed to create '%s'. HDFS did not return a redirect" % (path,)) # Now talk to the real thing. The redirect url already includes the params. client = self._make_client(next_url, self.security_enabled) headers = {'Content-Type': 'application/octet-stream'} return resource.Resource(client).invoke(method, data=data, headers=headers) def _get_redirect_url(self, webhdfs_ex): """Retrieve the redirect url from an exception object""" try: # The actual HttpError (307) is wrapped inside http_error = webhdfs_ex.get_parent_ex() if http_error is None: raise webhdfs_ex if http_error.code not in (301, 302, 303, 307): LOG.error("Response is not a redirect: %s" % (webhdfs_ex,)) raise webhdfs_ex return http_error.headers.getheader('location') except Exception, ex:
self._superuser = hdfs_superuser self._security_enabled = security_enabled self._ssl_cert_ca_verify = ssl_cert_ca_verify self._temp_dir = temp_dir self._umask = umask self._fs_defaultfs = fs_defaultfs self._logical_name = logical_name self._supergroup = hdfs_supergroup self._scheme = "" self._netloc = "" self._is_remote = False self._has_trash_support = True self._client = self._make_client(url, security_enabled, ssl_cert_ca_verify) self._root = resource.Resource(self._client) # To store user info self._thread_local = threading.local() LOG.debug( "Initializing Hadoop WebHdfs: %s (security: %s, superuser: %s)" % (self._url, self._security_enabled, self._superuser)) @classmethod def from_config(cls, hdfs_config): fs_defaultfs = hdfs_config.FS_DEFAULTFS.get() return cls(url=_get_service_url(hdfs_config), fs_defaultfs=fs_defaultfs, logical_name=hdfs_config.LOGICAL_NAME.get(),