def _credentials_override(f):
"""Provides special handling for credentials. It still calls _override().
If 'base64_password' in config is set, it will base64 decode it and returned in return value's 'password' field.
If 'rsa_password' in config is set, it will rsa decode it and returned in return value's 'password' field.
If 'base64_password' or 'rsa_password' is not set, it will fallback to 'password' in config.
"""
credentials = f()
rsa = rsa_file()
decoded_credentials = {
k: credentials.get(k)
for k in ('username', 'password', 'url', 'auth')
}
base64_password = credentials.get('base64_password')
rsa_password = credentials.get('rsa_password')
rsa_username = credentials.get('rsa_username')
if rsa_password is not None:
try:
b = bytes(rsa_password, 'utf-8')
text = base64.decodestring(b)
f = open(rsa, 'r')
pri_key = RSA.importKey(f.read())
cipher = PKCS1_v1_5.new(pri_key)
dsize = SHA.digest_size
sentinel = Random.new().read(15 + dsize)
decoded_credentials['password'] = cipher.decrypt(text, sentinel)
except Exception:
exception_type, exception, traceback = sys.exc_info()
msg = "ras_password for %s contains invalid rsa string: %s %s" % (
f.__name__, exception_type, exception)
raise BadUserConfigurationException(msg)
if rsa_username is not None:
try:
b = bytes(rsa_username, 'utf-8')
text = base64.decodestring(b)
f = open(rsa, 'r')
pri_key = RSA.importKey(f.read())
cipher = PKCS1_v1_5.new(pri_key)
dsize = SHA.digest_size
sentinel = Random.new().read(15 + dsize)
decoded_credentials['username'] = cipher.decrypt(text, sentinel)
except Exception:
exception_type, exception, traceback = sys.exc_info()
msg = "ras_username for %s contains invalid rsa string: %s %s" % (
f.__name__, exception_type, exception)
raise BadUserConfigurationException(msg)
if base64_password is not None:
try:
decoded_credentials['password'] = base64.b64decode(
base64_password).decode()
except Exception:
exception_type, exception, traceback = sys.exc_info()
msg = "base64_password for %s contains invalid base64 string: %s %s" % (
f.__name__, exception_type, exception)
raise BadUserConfigurationException(msg)
if decoded_credentials['auth'] is None:
decoded_credentials['auth'] = get_auth_value(
decoded_credentials['username'], decoded_credentials['password'])
return decoded_credentials
def _credentials_override(f):
"""Provides special handling for credentials. It still calls _override().
If 'base64_password' in config is set, it will base64 decode it and returned in return value's 'password' field.
If 'base64_password' is not set, it will fallback to 'password' in config.
"""
credentials = f()
base64_decoded_credentials = {
k: credentials.get(k)
for k in ('username', 'password', 'url', 'auth')
}
base64_password = credentials.get('base64_password')
if base64_password is not None:
try:
base64_decoded_credentials['password'] = base64.b64decode(
base64_password).decode()
except Exception:
exception_type, exception, traceback = sys.exc_info()
msg = "base64_password for %s contains invalid base64 string: %s %s" % (
f.__name__, exception_type, exception)
raise BadUserConfigurationException(msg)
if base64_decoded_credentials['auth'] is None:
base64_decoded_credentials['auth'] = get_auth_value(
base64_decoded_credentials['username'],
base64_decoded_credentials['password'])
return base64_decoded_credentials
def __init__(self, endpoint, headers, retry_policy):
self._endpoint = endpoint
self._headers = headers
self._retry_policy = retry_policy
if self._endpoint.auth == constants.AUTH_KERBEROS:
self._auth = HTTPKerberosAuth(**conf.kerberos_auth_configuration())
elif self._endpoint.auth == constants.AUTH_BASIC:
self._auth = (self._endpoint.username, self._endpoint.password)
elif self._endpoint.auth == constants.GOOGLE_AUTH:
credentials = (conf.google_auth_credentials())
self._auth = credentials
#Once we have credentials, we attach them to a transport. We use the transport to
#make authenticated requests: how does self._session work? Do I not have to use
#AuthorizedSession? Kerberos HTTPKerberosAuth attaches Kerberos Auth to Requests object
#so should self._auth = credentials or authed_session?
authed_session = AuthorizedSession(credentials)
elif self._endpoint.auth != constants.NO_AUTH:
raise BadUserConfigurationException(u"Unsupported auth %s" %
self._endpoint.auth)
self._session = requests.Session()
self.logger = SparkLog(u"ReliableHttpClient")
self.verify_ssl = not conf.ignore_ssl_errors()
if not self.verify_ssl:
self.logger.debug(
u"ATTENTION: Will ignore SSL errors. This might render you vulnerable to attacks."
)
requests.packages.urllib3.disable_warnings()
def initialize_auth(args):
"""Creates an authenticatior class instance for the given auth type
Args:
args (IPython.core.magics.namespace): The namespace object that is created from
parsing %spark magic command
Returns:
An instance of a valid Authenticator or None if args.auth is 'None'
Raises:
sparkmagic.livyclientlib.BadUserConfigurationException: if args.auth is not a valid
authenticator class.
"""
if args.auth is None:
auth = conf.get_auth_value(args.user, args.password)
else:
auth = args.auth
if auth == constants.NO_AUTH:
return None
else:
full_class = conf.authenticators().get(auth)
if full_class is None:
raise BadUserConfigurationException(u"Auth '{}' not supported".format(auth))
module, class_name = (full_class).rsplit('.', 1)
events_handler_module = importlib.import_module(module)
auth_class = getattr(events_handler_module, class_name)
return auth_class(args)
def __init__(self, endpoint, headers, retry_policy):
self._endpoint = endpoint
self._headers = headers
self._retry_policy = retry_policy
if self._endpoint.auth == constants.AUTH_KERBEROS:
if self._endpoint.krb_mutual_auth == constants.AUTH_KERBEROS_MUTUAL_REQ:
mutual_auth = REQUIRED
elif self._endpoint.krb_mutual_auth == constants.AUTH_KERBEROS_MUTUAL_OPT:
mutual_auth = OPTIONAL
elif self._endpoint.krb_mutual_auth == constants.AUTH_KERBEROS_MUTUAL_DIS:
mutual_auth = DISABLED
else:
mutual_auth = REQUIRED
if self._endpoint.krb_host_override == "":
hostname_override = None
else:
hostname_override = self._endpoint.krb_host_override
self._auth = HTTPKerberosAuth(mutual_authentication=mutual_auth,
hostname_override=hostname_override)
elif self._endpoint.auth == constants.AUTH_BASIC:
self._auth = (self._endpoint.username, self._endpoint.password)
elif self._endpoint.auth != constants.NO_AUTH:
raise BadUserConfigurationException(u"Unsupported auth %s" %
self._endpoint.auth)
self.logger = SparkLog(u"ReliableHttpClient")
self.verify_ssl = not conf.ignore_ssl_errors()
if not self.verify_ssl:
self.logger.debug(
u"ATTENTION: Will ignore SSL errors. This might render you vulnerable to attacks."
)
requests.packages.urllib3.disable_warnings()
def get_livy_kind(language):
if language == LANG_SCALA:
return SESSION_KIND_SPARK
elif language == LANG_PYTHON:
return SESSION_KIND_PYSPARK
elif language == LANG_R:
return SESSION_KIND_SPARKR
else:
raise BadUserConfigurationException("Cannot get session kind for {}.".format(language))
def _get_retry_policy():
policy = conf.retry_policy()
if policy == LINEAR_RETRY:
return LinearRetryPolicy(seconds_to_sleep=5, max_retries=5)
elif policy == CONFIGURABLE_RETRY:
return ConfigurableRetryPolicy(
retry_seconds_to_sleep_list=conf.retry_seconds_to_sleep_list(),
max_retries=conf.configurable_retry_policy_max_retries())
else:
raise BadUserConfigurationException(
u"Retry policy '{}' not supported".format(policy))
def test_no_credenntials_raises_bad_user_configuration_error():
no_credentials_exception = BadUserConfigurationException(
"Failed to obtain access token. Run `gcloud auth login` in your command line \
to authorize gcloud to access the Cloud Platform with Google user credentials to authenticate. Run `gcloud auth \
application-default login` acquire new user credentials to use for Application Default Credentials."
)
google_auth = GoogleAuth()
google_auth.project_widget.value = 'project_id'
google_auth.region_widget.value = 'region'
google_auth.cluster_widget.value = 'cluster_name'
google_auth.credentials = None
assert_raises(google_auth.update_with_widget_values(),
no_credentials_exception)
def __init__(self, retry_seconds_to_sleep_list, max_retries):
super(ConfigurableRetryPolicy, self).__init__(-1, max_retries)
# If user configured to an empty list, let's make this behave as
# a Linear Retry Policy by assigning a list of 1 element.
if len(retry_seconds_to_sleep_list) == 0:
retry_seconds_to_sleep_list = [5]
elif not all(n > 0 for n in retry_seconds_to_sleep_list):
raise BadUserConfigurationException(
u"All items in the list in your config need to be positive for configurable retry policy"
)
self.retry_seconds_to_sleep_list = retry_seconds_to_sleep_list
self._max_index = len(self.retry_seconds_to_sleep_list) - 1
def get_livy_kind(language):
if language == LANG_SCALA:
return SESSION_KIND_SPARK
elif language == LANG_PYTHON:
return SESSION_KIND_PYSPARK
elif language == LANG_PYTHON3:
# Starting with version 0.5.0-incubating, session kind “pyspark3” is removed,
# instead users require to set PYSPARK_PYTHON to python3 executable.
return SESSION_KIND_PYSPARK
elif language == LANG_R:
return SESSION_KIND_SPARKR
elif language == LANG_SQL:
return SESSION_KIND_SQL
else:
raise BadUserConfigurationException("Cannot get session kind for {}.".format(language))
def __init__(self, parsed_attributes=None):
self.callable_request = google.auth.transport.requests.Request()
self.scopes = [
'https://www.googleapis.com/auth/cloud-platform',
'https://www.googleapis.com/auth/userinfo.email'
]
self.credentialed_accounts, active_user_account = list_credentialed_user_accounts(
)
self.default_credentials_configured = application_default_credentials_configured(
)
if self.default_credentials_configured:
self.credentialed_accounts.append('default-credentials')
self.active_credentials = None
if parsed_attributes is not None:
if parsed_attributes.account in self.credentialed_accounts:
self.active_credentials = parsed_attributes.account
if self.active_credentials == 'default-credentials' and \
self.default_credentials_configured:
self.credentials, self.project = google.auth.default(
scopes=self.scopes)
else:
self.credentials, self.project = get_credentials_for_account(
self.active_credentials, self.scopes)
else:
new_exc = BadUserConfigurationException(
f"{parsed_attributes.account} is not a credentialed account. Run `gcloud "\
"auth login` in your command line to authorize gcloud to access the Cloud "\
"Platform with Google user credentials to authenticate. Run `gcloud auth "\
"application-default login` to acquire new user credentials to use for "\
"Application Default Credentials. Run `gcloud auth list` to see your credentialed "\
"accounts.")
raise new_exc
else:
if self.default_credentials_configured:
self.credentials, self.project = google.auth.default(
scopes=self.scopes)
self.active_credentials = 'default-credentials'
elif active_user_account is not None:
self.credentials, self.project = get_credentials_for_account(
active_user_account, self.scopes)
self.active_credentials = active_user_account
else:
self.credentials, self.project = None, None
Authenticator.__init__(self, parsed_attributes)
self.widgets = self.get_widgets(constants.WIDGET_WIDTH)
def update_with_widget_values(self):
"""Updates url to be the component gateway url of the cluster found with the project,
region, and cluster textbox widget values"""
no_credentials_exception = BadUserConfigurationException(
"Failed to obtain access token. Run `gcloud auth login` in your command line "\
"to authorize gcloud to access the Cloud Platform with Google user credentials to "\
"authenticate. Run `gcloud auth application-default login` acquire new user "\
"credentials to use for Application Default Credentials.")
if self.credentials is not None:
try:
self.initialize_credentials_with_auth_account_selection(
self.account_widget.v_model)
self.url, self.cluster_widget.v_model = get_component_gateway_url(
self.project_widget.v_model, self.region_widget.v_model,
self.cluster_widget.v_model, self.credentials)
except:
raise
else:
raise no_credentials_exception
def list_credentialed_user_accounts():
"""Load all of user's credentialed accounts with ``gcloud auth list`` command.
Returns:
Sequence[str]: each value is a str of one of the users credentialed accounts
Raises:
sparkmagic.livyclientlib.BadUserConfigurationException: if gcloud cannot be invoked
"""
accounts_json = ""
if os.name == "nt":
command = constants.CLOUD_SDK_WINDOWS_COMMAND
else:
command = constants.CLOUD_SDK_POSIX_COMMAND
try:
command = (
command, ) + constants.CLOUD_SDK_USER_CREDENTIALED_ACCOUNTS_COMMAND
# run `gcloud auth list` command
accounts_json = subprocess.check_output(command,
stderr=subprocess.STDOUT)
account_objects = json.loads(accounts_json)
credentialed_accounts = list()
active_account = None
#convert account dictionaries with status and account keys to a list of accounts
for account in account_objects:
try:
# if the account does not have an access token we don't add it to the account
# dropdown
_cloud_sdk.get_auth_access_token(account['account'])
# service accounts will be added later with 'default-credentials'
get_credentials_for_account(account['account'])
if account['status'] == 'ACTIVE':
active_account = account['account']
credentialed_accounts.append(account['account'])
# when`gcloud auth print-access-token --account=account` fails we don't add it to
# credentialed_accounts list that populates account dropdown widget
except:
pass
return credentialed_accounts, active_account
except Exception as caught_exc:
new_exc = BadUserConfigurationException("Gcloud cannot be invoked.")
raise new_exc from caught_exc
def __init__(self, endpoint, headers, retry_policy):
self._endpoint = endpoint
self._headers = headers
self._retry_policy = retry_policy
if self._endpoint.auth == constants.AUTH_KERBEROS:
self._auth = HTTPKerberosAuth(**conf.kerberos_auth_configuration())
elif self._endpoint.auth == constants.AUTH_BASIC:
self._auth = (self._endpoint.username, self._endpoint.password)
elif self._endpoint.auth != constants.NO_AUTH:
raise BadUserConfigurationException(u"Unsupported auth %s" %
self._endpoint.auth)
self._session = requests.Session()
self.logger = SparkLog(u"ReliableHttpClient")
self.verify_ssl = not conf.ignore_ssl_errors()
if not self.verify_ssl:
self.logger.debug(
u"ATTENTION: Will ignore SSL errors. This might render you vulnerable to attacks."
)
requests.packages.urllib3.disable_warnings()