def copy_primary_beams(client):
BANDS = 'lu' # The only bands implemented so far - update as more are added
urls = {
f'https://sdpmodels.kat.ac.za/primary_beam/current/{group}/{antenna}/{band}.alias'
for group in ['individual', 'cohort']
for antenna in [f'm{i:03d}' for i in range(64)] for band in BANDS
}
urls.update({
f'https://sdpmodels.kat.ac.za/primary_beam/current/cohort/{cohort}/{band}.alias'
for cohort in ['meerkat'] for band in BANDS
})
done = set()
with requests.Session() as session:
while urls:
url = urls.pop()
done.add(url)
with session.get(url, timeout=20) as resp:
resp.raise_for_status()
client.put_object(Bucket='models',
ContentType=resp.headers['Content-type'],
Key=urlsplit(url).path[1:],
Body=resp.content)
if url.endswith('.alias'):
rel_path = resp.text.strip()
rel_parts = urlsplit(rel_path)
assert not rel_parts.scheme and not rel_parts.netloc
new_url = urljoin(resp.url, rel_path)
if new_url not in done:
urls.add(new_url)
def _get_idp_response(
self, saml_config: Dict[str, Any],
auth: requests.auth.AuthBase) -> requests.models.Response:
idp_url = saml_config["idp_url"]
self.log.info("idp_url= %s", idp_url)
session = requests.Session()
# Configurable Retry when querying the IDP endpoint
if "idp_request_retry_kwargs" in saml_config:
idp_request_retry_kwargs = saml_config["idp_request_retry_kwargs"]
self.log.info("idp_request_retry_kwargs= %s",
idp_request_retry_kwargs)
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
retry_strategy = Retry(**idp_request_retry_kwargs)
adapter = HTTPAdapter(max_retries=retry_strategy)
session.mount("https://", adapter)
session.mount("http://", adapter)
idp_request_kwargs = {}
if "idp_request_kwargs" in saml_config:
idp_request_kwargs = saml_config["idp_request_kwargs"]
idp_response = session.get(idp_url, auth=auth, **idp_request_kwargs)
idp_response.raise_for_status()
return idp_response
def _refresh_credentials():
session = get_session()
creds = session.get("{url}/api/auth/get_credentials".format(
url=get_registry_url())).json()
result = {
'access_key': creds['AccessKeyId'],
'secret_key': creds['SecretAccessKey'],
'token': creds['SessionToken'],
'expiry_time': creds['Expiration']
}
_save_credentials(result)
return result
def file_download(request):
if not(BaseView.is_csrf_valid_static(request)):
return JSONResponse(status=400, message="missing CSRF token")
session = request.session
if session.get('file_cache'):
(filename, mime_type, contents) = session['file_cache']
# Clean the session information regrading the new keypair
del session['file_cache']
response = Response(content_type=mime_type)
response.body = str(contents)
response.content_disposition = 'attachment; filename="{name}"'.format(name=filename)
response.cache_control = 'no-store'
response.pragma = 'no-cache'
return response
# no file found ...
# this isn't handled on on client anyway, so we can return pretty much anything
return Response(body='BaseView:file not found', status=500)
def _get_credentials(self, saml_url, username, password, region):
session = requests.Session()
formresponse = session.get(saml_url, verify=True)
saml_submit_url = formresponse.url
formsoup = BeautifulSoup(formresponse.text.decode('utf-8'))
payload = {}
for inputtag in formsoup.find_all(re.compile('(INPUT|input)')):
name = inputtag.get('name', '')
value = inputtag.get('value', '')
if 'username' in name.lower():
payload[name] = username
elif 'email' in name.lower():
payload[name] = username
elif 'password' in name.lower():
payload[name] = password
else:
payload[name] = value
for inputtag in formsoup.find_all(re.compile('(FORM|form)')):
action = inputtag.get('action')
if action:
parsedurl = urlparse(saml_url)
saml_submit_url = parsedurl.scheme + '://' + parsedurl.netloc + action
response = session.post(
saml_submit_url, params=payload, verify=True)
del username
del password
soup = BeautifulSoup(response.text.decode('utf-8'))
assertion = None
for inputtag in soup.find_all('input'):
if(inputtag.get('name') == 'SAMLResponse'):
assertion = inputtag.get('value')
if assertion is None:
print('Response did not contain a valid SAML assertion!')
return 1
awsroles = []
root = ET.fromstring(base64.b64decode(assertion))
for saml2attribute in root.iter('{urn:oasis:names:tc:SAML:2.0:assertion}Attribute'):
if (saml2attribute.get('Name') == 'https://aws.amazon.com/SAML/Attributes/Role'):
for saml2attributevalue in saml2attribute.iter('{urn:oasis:names:tc:SAML:2.0:assertion}AttributeValue'):
awsroles.append(saml2attributevalue.text)
for awsrole in awsroles:
chunks = awsrole.split(',')
if'saml-provider' in chunks[0]:
newawsrole = chunks[1] + ',' + chunks[0]
index = awsroles.index(awsrole)
awsroles.insert(index, newawsrole)
awsroles.remove(awsrole)
if len(awsroles) > 1:
i = 0
print('Please choose the role you would like to assume:')
for awsrole in awsroles:
print('[{}]: '.format(awsrole.split(',')[0]))
i += 1
selectedroleindex = raw_input()
print('Selection: {}'.format(selectedroleindex))
# Basic sanity check of input
if int(selectedroleindex) > (len(awsroles) - 1):
print('You selected an invalid role index, please try again')
return 1
role_arn = awsroles[int(selectedroleindex)].split(',')[0]
principal_arn = awsroles[int(selectedroleindex)].split(',')[1]
else:
role_arn = awsroles[0].split(',')[0]
principal_arn = awsroles[0].split(',')[1]
sts = botocore.session.create_client('sts')
token = sts.assume_role_with_saml(role_arn, principal_arn, assertion)
credentials = {}
credentials['aws_access_key_id'] = token.credentials.access_key
credentials['aws_secret_access_key'] = token.credentials.secret_key
credentials['aws_session_token'] = token.credentials.session_token
return credentials
def saml_login(profile,
region,
url,
user,
password=None,
role=None,
print_env_vars=False,
overwrite_default_credentials=False):
session = requests.Session()
response = session.get(url)
keyring_key = 'aws-minion.saml'
password = password or keyring.get_password(keyring_key, user)
if not password:
password = click.prompt('Password', hide_input=True)
with Action('Authenticating against {url}..', **vars()) as act:
# NOTE: parameters are hardcoded for Shibboleth IDP
data = {'j_username': user, 'j_password': password, 'submit': 'Login'}
response2 = session.post(response.url, data=data)
saml_xml = get_saml_response(response2.text)
if not saml_xml:
act.error('LOGIN FAILED')
click.secho(
'SAML login with user "{}" failed, please check your username and password.\n'
.format(user) +
'You might need to change the password in your keyring (e.g. Mac OS X keychain) '
+ 'or use the "--password" option.',
bold=True,
fg='blue')
return
url = get_form_action(response2.text)
encoded_xml = codecs.encode(saml_xml.encode('utf-8'), 'base64')
response3 = session.post(url, data={'SAMLResponse': encoded_xml})
account_names = get_account_names(response3.text)
keyring.set_password(keyring_key, user, password)
with Action('Checking SAML roles..') as act:
roles = get_roles(saml_xml)
if not roles:
act.error('NO VALID ROLE FOUND')
return
if len(roles) == 1:
provider_arn, role_arn = roles[0]
elif role:
matching_roles = [
_role for _role in roles
if role in get_role_label(_role, account_names)
]
if not matching_roles or len(matching_roles) > 1:
raise click.UsageError(
'Given role (--role) was not found or not unique')
provider_arn, role_arn = matching_roles[0]
else:
roles.sort()
provider_arn, role_arn = choice(
'Multiple roles found, please select one.',
[(r, get_role_label(r, account_names)) for r in roles])
with Action('Assuming role "{role_label}"..',
role_label=get_role_label((provider_arn, role_arn),
account_names)):
saml_assertion = codecs.encode(saml_xml.encode('utf-8'),
'base64').decode('ascii').replace(
'\n', '')
# botocore NEEDS some credentials, but does not care about their actual values
os.environ['AWS_ACCESS_KEY_ID'] = 'fake123'
os.environ['AWS_SECRET_ACCESS_KEY'] = 'fake123'
try:
session = botocore.session.get_session()
sts = session.get_service('sts')
operation = sts.get_operation('AssumeRoleWithSAML')
endpoint = sts.get_endpoint(region)
endpoint._signature_version = None
http_response, response_data = operation.call(
endpoint,
role_arn=role_arn,
principal_arn=provider_arn,
SAMLAssertion=saml_assertion)
finally:
del os.environ['AWS_ACCESS_KEY_ID']
del os.environ['AWS_SECRET_ACCESS_KEY']
key_id = response_data['Credentials']['AccessKeyId']
secret = response_data['Credentials']['SecretAccessKey']
session_token = response_data['Credentials']['SessionToken']
if print_env_vars:
# different AWS SDKs expect either AWS_SESSION_TOKEN or AWS_SECURITY_TOKEN, so set both
click.secho(dedent('''\
# environment variables with temporary AWS credentials:
export AWS_ACCESS_KEY_ID="{key_id}"
export AWS_SECRET_ACCESS_KEY="{secret}"
export AWS_SESSION_TOKEN="{session_token}")
export AWS_SECURITY_TOKEN="{session_token}"''').format(**vars()),
fg='blue')
profiles_to_write = set([profile])
if overwrite_default_credentials:
profiles_to_write.add('default')
with Action('Writing temporary AWS credentials..'):
for prof in profiles_to_write:
write_aws_credentials(prof, key_id, secret, session_token)
def saml_login(profile, region, url, user, password=None, role=None, print_env_vars=False,
overwrite_default_credentials=False):
session = requests.Session()
response = session.get(url)
keyring_key = 'aws-minion.saml'
password = password or keyring.get_password(keyring_key, user)
if not password:
password = click.prompt('Password', hide_input=True)
with Action('Authenticating against {url}..', **vars()) as act:
# NOTE: parameters are hardcoded for Shibboleth IDP
data = {'j_username': user, 'j_password': password, 'submit': 'Login'}
response2 = session.post(response.url, data=data)
saml_xml = get_saml_response(response2.text)
if not saml_xml:
act.error('LOGIN FAILED')
click.secho('SAML login with user "{}" failed, please check your username and password.\n'.format(user) +
'You might need to change the password in your keyring (e.g. Mac OS X keychain) ' +
'or use the "--password" option.', bold=True, fg='blue')
return
url = get_form_action(response2.text)
encoded_xml = codecs.encode(saml_xml.encode('utf-8'), 'base64')
response3 = session.post(url, data={'SAMLResponse': encoded_xml})
account_names = get_account_names(response3.text)
keyring.set_password(keyring_key, user, password)
with Action('Checking SAML roles..') as act:
roles = get_roles(saml_xml)
if not roles:
act.error('NO VALID ROLE FOUND')
return
if len(roles) == 1:
provider_arn, role_arn = roles[0]
elif role:
matching_roles = [_role for _role in roles if role in get_role_label(_role, account_names)]
if not matching_roles or len(matching_roles) > 1:
raise click.UsageError('Given role (--role) was not found or not unique')
provider_arn, role_arn = matching_roles[0]
else:
roles.sort()
provider_arn, role_arn = choice('Multiple roles found, please select one.',
[(r, get_role_label(r, account_names)) for r in roles])
with Action('Assuming role "{role_label}"..', role_label=get_role_label((provider_arn, role_arn), account_names)):
saml_assertion = codecs.encode(saml_xml.encode('utf-8'), 'base64').decode('ascii').replace('\n', '')
# botocore NEEDS some credentials, but does not care about their actual values
os.environ['AWS_ACCESS_KEY_ID'] = 'fake123'
os.environ['AWS_SECRET_ACCESS_KEY'] = 'fake123'
try:
session = botocore.session.get_session()
sts = session.get_service('sts')
operation = sts.get_operation('AssumeRoleWithSAML')
endpoint = sts.get_endpoint(region)
endpoint._signature_version = None
http_response, response_data = operation.call(endpoint, role_arn=role_arn, principal_arn=provider_arn,
SAMLAssertion=saml_assertion)
finally:
del os.environ['AWS_ACCESS_KEY_ID']
del os.environ['AWS_SECRET_ACCESS_KEY']
key_id = response_data['Credentials']['AccessKeyId']
secret = response_data['Credentials']['SecretAccessKey']
session_token = response_data['Credentials']['SessionToken']
if print_env_vars:
# different AWS SDKs expect either AWS_SESSION_TOKEN or AWS_SECURITY_TOKEN, so set both
click.secho(dedent('''\
# environment variables with temporary AWS credentials:
export AWS_ACCESS_KEY_ID="{key_id}"
export AWS_SECRET_ACCESS_KEY="{secret}"
export AWS_SESSION_TOKEN="{session_token}")
export AWS_SECURITY_TOKEN="{session_token}"''').format(**vars()), fg='blue')
profiles_to_write = set([profile])
if overwrite_default_credentials:
profiles_to_write.add('default')
with Action('Writing temporary AWS credentials..'):
for prof in profiles_to_write:
write_aws_credentials(prof, key_id, secret, session_token)