def _fetch_saml_assertion_using_http_spegno_auth(self,
saml_config: Dict[str,
Any]):
import requests
# requests_gssapi will need paramiko > 2.6 since you'll need
# 'gssapi' not 'python-gssapi' from PyPi.
# https://github.com/paramiko/paramiko/pull/1311
import requests_gssapi
from lxml import etree
idp_url = saml_config["idp_url"]
self.log.info("idp_url= %s", idp_url)
idp_request_kwargs = saml_config["idp_request_kwargs"]
auth = requests_gssapi.HTTPSPNEGOAuth()
if 'mutual_authentication' in saml_config:
mutual_auth = saml_config['mutual_authentication']
if mutual_auth == 'REQUIRED':
auth = requests_gssapi.HTTPSPNEGOAuth(requests_gssapi.REQUIRED)
elif mutual_auth == 'OPTIONAL':
auth = requests_gssapi.HTTPSPNEGOAuth(requests_gssapi.OPTIONAL)
elif mutual_auth == 'DISABLED':
auth = requests_gssapi.HTTPSPNEGOAuth(requests_gssapi.DISABLED)
else:
raise NotImplementedError(
f'mutual_authentication={mutual_auth} in Connection {self.conn.conn_id} Extra.'
'Currently "REQUIRED", "OPTIONAL" and "DISABLED" are supported.'
'(Exclude this setting will default to HTTPSPNEGOAuth() ).'
)
# Query the IDP
idp_reponse = requests.get(idp_url, auth=auth, **idp_request_kwargs)
idp_reponse.raise_for_status()
# Assist with debugging. Note: contains sensitive info!
xpath = saml_config['saml_response_xpath']
log_idp_response = 'log_idp_response' in saml_config and saml_config[
'log_idp_response']
if log_idp_response:
self.log.warning(
'The IDP response contains sensitive information,'
' but log_idp_response is ON (%s).',
log_idp_response,
)
self.log.info('idp_reponse.content= %s', idp_reponse.content)
self.log.info('xpath= %s', xpath)
# Extract SAML Assertion from the returned HTML / XML
xml = etree.fromstring(idp_reponse.content)
saml_assertion = xml.xpath(xpath)
if isinstance(saml_assertion, list):
if len(saml_assertion) == 1:
saml_assertion = saml_assertion[0]
if not saml_assertion:
raise ValueError('Invalid SAML Assertion')
return saml_assertion
def _post(url, data, headers):
try:
# try to authenticate using opportunistic auth first
return requests.post(url,
data=data,
headers=headers,
auth=requests_gssapi.HTTPSPNEGOAuth(
opportunistic_auth=True))
except requests_gssapi.exceptions.SPNEGOExchangeError:
return requests.post(url,
data=data,
headers=headers,
auth=requests_gssapi.HTTPSPNEGOAuth())
def _login_kerberos(self):
"""
private function, use login_kerberos instead
"""
if isinstance(requests_gssapi, ImportError):
raise requests_gssapi
login_url = 'https://{0}/ipa/session/login_kerberos'.format(
self._current_host)
headers = {'Referer': 'https://{0}/ipa'.format(self._current_host)}
response = self._session.post(
login_url,
headers=headers,
verify=self._verify_ssl,
auth=requests_gssapi.HTTPSPNEGOAuth(),
)
if not response.ok:
raise Unauthorized(response.text)
self.log.info(
'Successfully logged to {0} using Kerberos credentials.'.format(
self._current_host))
return AuthenticatedSession(self, logged_in=True)
def _upload_source(cls,
url,
package,
filename,
hashtype,
hsh,
auth=requests_gssapi.HTTPSPNEGOAuth()):
class ChunkedData(object):
def __init__(self, check_only, chunksize=8192):
self.check_only = check_only
self.chunksize = chunksize
self.start = time.time()
self.uploaded = False
fields = [
('name', package),
('{}sum'.format(hashtype), hsh),
]
if check_only:
fields.append(('filename', filename))
else:
with open(filename, 'rb') as f:
rf = RequestField('file', f.read(), filename)
rf.make_multipart()
fields.append(rf)
self.data, content_type = encode_multipart_formdata(fields)
self.headers = {'Content-Type': content_type}
def __iter__(self):
if self.uploaded:
# ensure the progressbar is shown only once (HTTPSPNEGOAuth causes second request)
yield self.data
else:
totalsize = len(self.data)
for offset in range(0, totalsize, self.chunksize):
transferred = min(offset + self.chunksize, totalsize)
if not self.check_only:
DownloadHelper.progress(totalsize, transferred,
self.start)
yield self.data[offset:transferred]
self.uploaded = True
def post(check_only=False):
cd = ChunkedData(check_only)
r = requests.post(url, data=cd, headers=cd.headers, auth=auth)
if not 200 <= r.status_code < 300:
raise LookasideCacheError(r.reason)
return r.content
state = post(check_only=True)
if state.strip() == b'Available':
# already uploaded
return
logger.info('Uploading %s to lookaside cache', filename)
try:
post()
finally:
sys.stdout.write('\n')
sys.stdout.flush()
def _login_gssapi(self):
"""Login using kerberos credentials (uses gssapi)."""
login_url = urlparse.urljoin(self._hub_url, "auth/krb5login/")
# read default values from settings
principal = self._conf.get("KRB_PRINCIPAL")
keytab = self._conf.get("KRB_KEYTAB")
service = self._conf.get("KRB_SERVICE")
realm = self._conf.get("KRB_REALM")
ccache = self._conf.get("KRB_CCACHE")
import requests
import gssapi
import requests_gssapi
request_args = {}
# NOTE behavior difference from hub proxy overall:
# HubProxy by default DOES NOT verify https connections :(
# See the constructor. It could be repeated here by defaulting verify to False,
# but let's not do that, instead you must have an unbroken SSL setup to
# use this auth method.
if self._conf.get("CA_CERT"):
request_args["verify"] = self._conf["CA_CERT"]
server_name = self.get_server_principal(service=service, realm=realm)
server_name = gssapi.Name(server_name,
gssapi.NameType.kerberos_principal)
auth_args = {
"target_name": server_name,
}
if principal is not None:
if keytab is None:
raise ImproperlyConfigured(
"Cannot specify a principal without a keytab")
name = gssapi.Name(principal, gssapi.NameType.kerberos_principal)
store = {"client_keytab": keytab}
if ccache is not None:
store["ccache"] = "FILE:" + ccache
auth_args["creds"] = gssapi.Credentials(name=name,
store=store,
usage="initiate")
# We only do one request, but a Session is used to allow requests to write
# the new session ID into the cookiejar.
with requests.Session() as s:
s.cookies = self._transport.cookiejar
response = s.get(login_url,
auth=requests_gssapi.HTTPSPNEGOAuth(**auth_args),
allow_redirects=False,
**request_args)
self._logger and self._logger.debug("Login response: %s %s", response,
response.headers)
response.raise_for_status()
def test_opportunistic_auth(self):
with patch.multiple("gssapi.SecurityContext", __init__=fake_init,
step=fake_resp):
auth = requests_gssapi.HTTPSPNEGOAuth(opportunistic_auth=True)
request = requests.Request(url="http://www.example.org")
auth.__call__(request)
self.assertTrue('Authorization' in request.headers)
self.assertEqual(request.headers.get('Authorization'),
b64_negotiate_response)
def test_target_name(self):
with patch.multiple("gssapi.SecurityContext", __init__=fake_init,
step=fake_resp):
response = requests.Response()
response.url = "http://www.example.org/"
response.headers = {'www-authenticate': b64_negotiate_token}
host = urlparse(response.url).hostname
auth = requests_gssapi.HTTPSPNEGOAuth(
target_name="*****@*****.**")
auth.generate_request_header(response, host)
fake_init.assert_called_with(
name=gssapi_name("*****@*****.**"),
usage="initiate", flags=gssflags, creds=None)
fake_resp.assert_called_with(b"token")
def test_explicit_creds(self):
with patch.multiple("gssapi.Credentials", __new__=fake_creds), \
patch.multiple("gssapi.SecurityContext", __init__=fake_init,
step=fake_resp):
response = requests.Response()
response.url = "http://www.example.org/"
response.headers = {'www-authenticate': b64_negotiate_token}
host = urlparse(response.url).hostname
creds = gssapi.Credentials()
auth = requests_gssapi.HTTPSPNEGOAuth(creds=creds)
auth.generate_request_header(response, host)
fake_init.assert_called_with(
name=gssapi_name("*****@*****.**"),
usage="initiate", flags=gssflags, creds=b"fake creds")
fake_resp.assert_called_with(b"token")
def set_gssapi_auth(self, **kwargs):
if requests_gssapi is None:
raise ImportError('requests_gssapi')
self.session.auth = requests_gssapi.HTTPSPNEGOAuth(**kwargs)
def _assume_role_with_saml(self, sts_client: boto3.client,
extra_config: dict, role_arn: str,
assume_role_kwargs: dict):
saml_config = extra_config['assume_role_with_saml']
principal_arn = saml_config['principal_arn']
idp_url = saml_config["idp_url"]
self.log.info("idp_url= %s", idp_url)
idp_request_kwargs = saml_config["idp_request_kwargs"]
idp_auth_method = saml_config['idp_auth_method']
if idp_auth_method == 'http_spegno_auth':
# requests_gssapi will need paramiko > 2.6 since you'll need
# 'gssapi' not 'python-gssapi' from PyPi.
# https://github.com/paramiko/paramiko/pull/1311
import requests_gssapi
auth = requests_gssapi.HTTPSPNEGOAuth()
if 'mutual_authentication' in saml_config:
mutual_auth = saml_config['mutual_authentication']
if mutual_auth == 'REQUIRED':
auth = requests_gssapi.HTTPSPNEGOAuth(
requests_gssapi.REQUIRED)
elif mutual_auth == 'OPTIONAL':
auth = requests_gssapi.HTTPSPNEGOAuth(
requests_gssapi.OPTIONAL)
elif mutual_auth == 'DISABLED':
auth = requests_gssapi.HTTPSPNEGOAuth(
requests_gssapi.DISABLED)
else:
raise NotImplementedError(
f'mutual_authentication={mutual_auth} in Connection {self.aws_conn_id} Extra.'
'Currently "REQUIRED", "OPTIONAL" and "DISABLED" are supported.'
'(Exclude this setting will default to HTTPSPNEGOAuth() ).'
)
# Query the IDP
import requests
idp_reponse = requests.get(idp_url,
auth=auth,
**idp_request_kwargs)
idp_reponse.raise_for_status()
# Assist with debugging. Note: contains sensitive info!
xpath = saml_config['saml_response_xpath']
log_idp_response = 'log_idp_response' in saml_config and saml_config[
'log_idp_response']
if log_idp_response:
self.log.warning(
'The IDP response contains sensitive information,'
' but log_idp_response is ON (%s).', log_idp_response)
self.log.info('idp_reponse.content= %s', idp_reponse.content)
self.log.info('xpath= %s', xpath)
# Extract SAML Assertion from the returned HTML / XML
from lxml import etree
xml = etree.fromstring(idp_reponse.content)
saml_assertion = xml.xpath(xpath)
if isinstance(saml_assertion, list):
if len(saml_assertion) == 1:
saml_assertion = saml_assertion[0]
if not saml_assertion:
raise ValueError('Invalid SAML Assertion')
else:
raise NotImplementedError(
f'idp_auth_method={idp_auth_method} in Connection {self.aws_conn_id} Extra.'
'Currently only "http_spegno_auth" is supported, and must be specified.'
)
self.log.info("Doing sts_client.assume_role_with_saml to role_arn=%s",
role_arn)
return sts_client.assume_role_with_saml(RoleArn=role_arn,
PrincipalArn=principal_arn,
SAMLAssertion=saml_assertion,
**assume_role_kwargs)
def _upload_source(
cls,
url,
package,
source_dir,
filename,
hashtype,
hsh,
auth=requests_gssapi.HTTPSPNEGOAuth(opportunistic_auth=True)):
class ChunkedData:
def __init__(self, check_only, chunksize=8192):
self.check_only = check_only
self.chunksize = chunksize
self.start = time.time()
fields = [
('name', package),
('{}sum'.format(hashtype), hsh),
]
if check_only:
fields.append(('filename', filename))
else:
fields.append(
('mtime', str(int(os.stat(filename).st_mtime))))
with open(path, 'rb') as f:
rf = RequestField('file', f.read(), filename)
rf.make_multipart()
fields.append(rf)
self.data, content_type = encode_multipart_formdata(fields)
self.headers = {'Content-Type': content_type}
def __iter__(self):
totalsize = len(self.data)
for offset in range(0, totalsize, self.chunksize):
transferred = min(offset + self.chunksize, totalsize)
if not self.check_only:
DownloadHelper.progress(totalsize, transferred,
self.start)
yield self.data[offset:transferred]
class FakeProgress(threading.Thread):
def __init__(self, check_only, interval=0.2):
self.check_only = check_only
self.interval = interval
self.stop_event = threading.Event()
super().__init__()
def run(self):
if self.check_only:
return
n = 0
start = time.time()
while not self.stop_event.is_set():
DownloadHelper.progress(-1,
n * 256 * 1024,
start,
show_size=False)
n += 1
self.stop_event.wait(self.interval)
def stop(self):
self.stop_event.set()
super().join()
def post(check_only=False):
cd = ChunkedData(check_only)
if 'src.fedoraproject.org' in url:
# src.fedoraproject.org seems to have trouble with chunked requests, don't even try
fp = FakeProgress(check_only)
fp.start()
try:
r = requests.post(url,
data=cd.data,
headers=cd.headers,
auth=auth)
finally:
fp.stop()
else:
r = requests.post(url, data=cd, headers=cd.headers, auth=auth)
if not 200 <= r.status_code < 300:
raise LookasideCacheError('{0}: {1}'.format(
r.reason, r.text.strip()))
return r.content
path = os.path.join(source_dir, filename)
try:
state = post(check_only=True)
except (requests.exceptions.ConnectionError, LookasideCacheError) as e:
# just log the error and bail out
logger.error('Attempt to upload to lookaside cache failed: %s',
str(e))
return
if state.strip() == b'Available':
# already uploaded
logger.info(
'%s is already present in lookaside cache, not uploading',
path)
return
logger.info('Uploading %s to lookaside cache', path)
try:
try:
post()
finally:
sys.stdout.write('\n')
sys.stdout.flush()
except (requests.exceptions.ConnectionError, LookasideCacheError) as e:
# Skip error, the rebase can continue even after a failed upload
logger.error("Upload to lookaside cache failed: %s", str(e))
def _login_gssapi_common(self, login_url, force_service, allow_redirects):
"""Common authentication logic for auth methods using gssapi.
- if force_service is True, client always calculates a service principal up-front
even if no KRB_SERVICE was set in config. Breaks OIDC since we are not actually
doing gssapi with the kobo hub.
- if allow_redirects is True, authentication is allowed to follow redirects. Required
in the OIDC case.
"""
# read default values from settings
principal = self._conf.get("KRB_PRINCIPAL")
keytab = self._conf.get("KRB_KEYTAB")
service = self._conf.get("KRB_SERVICE")
realm = self._conf.get("KRB_REALM")
ccache = self._conf.get("KRB_CCACHE")
import requests
import gssapi
import requests_gssapi
request_args = {}
# NOTE behavior difference from hub proxy overall:
# HubProxy by default DOES NOT verify https connections :(
# See the constructor. It could be repeated here by defaulting verify to False,
# but let's not do that, instead you must have an unbroken SSL setup to
# use this auth method.
if self._conf.get("CA_CERT"):
request_args["verify"] = self._conf["CA_CERT"]
auth_args = {"mutual_authentication": requests_gssapi.OPTIONAL}
if service or force_service:
if realm is None:
# let gssapi select the correct realm (according to system configuration)
if service is None:
service = "HTTP"
server_name = '%s@%s' % (service, self.get_hub_hostname())
server_name = gssapi.Name(server_name,
gssapi.NameType.hostbased_service)
else:
server_name = self.get_server_principal(service=service,
realm=realm)
server_name = gssapi.Name(server_name,
gssapi.NameType.kerberos_principal)
auth_args["target_name"] = server_name
if principal is not None:
if keytab is None:
raise ImproperlyConfigured(
"Cannot specify a principal without a keytab")
name = gssapi.Name(principal, gssapi.NameType.kerberos_principal)
store = {"client_keytab": keytab}
if ccache is not None:
store["ccache"] = "FILE:" + ccache
auth_args["creds"] = gssapi.Credentials(name=name,
store=store,
usage="initiate")
# We only do one request, but a Session is used to allow requests to write
# the new session ID into the cookiejar.
with requests.Session() as s:
s.cookies = self._transport.cookiejar
response = s.get(login_url,
auth=requests_gssapi.HTTPSPNEGOAuth(**auth_args),
allow_redirects=allow_redirects,
**request_args)
self._logger and self._logger.debug("Login response: %s %s", response,
response.headers)
response.raise_for_status()
