def copy_primary_beams(client): BANDS = 'lu' # The only bands implemented so far - update as more are added urls = { f'https://sdpmodels.kat.ac.za/primary_beam/current/{group}/{antenna}/{band}.alias' for group in ['individual', 'cohort'] for antenna in [f'm{i:03d}' for i in range(64)] for band in BANDS } urls.update({ f'https://sdpmodels.kat.ac.za/primary_beam/current/cohort/{cohort}/{band}.alias' for cohort in ['meerkat'] for band in BANDS }) done = set() with requests.Session() as session: while urls: url = urls.pop() done.add(url) with session.get(url, timeout=20) as resp: resp.raise_for_status() client.put_object(Bucket='models', ContentType=resp.headers['Content-type'], Key=urlsplit(url).path[1:], Body=resp.content) if url.endswith('.alias'): rel_path = resp.text.strip() rel_parts = urlsplit(rel_path) assert not rel_parts.scheme and not rel_parts.netloc new_url = urljoin(resp.url, rel_path) if new_url not in done: urls.add(new_url)
def _get_idp_response( self, saml_config: Dict[str, Any], auth: requests.auth.AuthBase) -> requests.models.Response: idp_url = saml_config["idp_url"] self.log.info("idp_url= %s", idp_url) session = requests.Session() # Configurable Retry when querying the IDP endpoint if "idp_request_retry_kwargs" in saml_config: idp_request_retry_kwargs = saml_config["idp_request_retry_kwargs"] self.log.info("idp_request_retry_kwargs= %s", idp_request_retry_kwargs) from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.retry import Retry retry_strategy = Retry(**idp_request_retry_kwargs) adapter = HTTPAdapter(max_retries=retry_strategy) session.mount("https://", adapter) session.mount("http://", adapter) idp_request_kwargs = {} if "idp_request_kwargs" in saml_config: idp_request_kwargs = saml_config["idp_request_kwargs"] idp_response = session.get(idp_url, auth=auth, **idp_request_kwargs) idp_response.raise_for_status() return idp_response
def _refresh_credentials(): session = get_session() creds = session.get("{url}/api/auth/get_credentials".format( url=get_registry_url())).json() result = { 'access_key': creds['AccessKeyId'], 'secret_key': creds['SecretAccessKey'], 'token': creds['SessionToken'], 'expiry_time': creds['Expiration'] } _save_credentials(result) return result
def file_download(request): if not(BaseView.is_csrf_valid_static(request)): return JSONResponse(status=400, message="missing CSRF token") session = request.session if session.get('file_cache'): (filename, mime_type, contents) = session['file_cache'] # Clean the session information regrading the new keypair del session['file_cache'] response = Response(content_type=mime_type) response.body = str(contents) response.content_disposition = 'attachment; filename="{name}"'.format(name=filename) response.cache_control = 'no-store' response.pragma = 'no-cache' return response # no file found ... # this isn't handled on on client anyway, so we can return pretty much anything return Response(body='BaseView:file not found', status=500)
def _get_credentials(self, saml_url, username, password, region): session = requests.Session() formresponse = session.get(saml_url, verify=True) saml_submit_url = formresponse.url formsoup = BeautifulSoup(formresponse.text.decode('utf-8')) payload = {} for inputtag in formsoup.find_all(re.compile('(INPUT|input)')): name = inputtag.get('name', '') value = inputtag.get('value', '') if 'username' in name.lower(): payload[name] = username elif 'email' in name.lower(): payload[name] = username elif 'password' in name.lower(): payload[name] = password else: payload[name] = value for inputtag in formsoup.find_all(re.compile('(FORM|form)')): action = inputtag.get('action') if action: parsedurl = urlparse(saml_url) saml_submit_url = parsedurl.scheme + '://' + parsedurl.netloc + action response = session.post( saml_submit_url, params=payload, verify=True) del username del password soup = BeautifulSoup(response.text.decode('utf-8')) assertion = None for inputtag in soup.find_all('input'): if(inputtag.get('name') == 'SAMLResponse'): assertion = inputtag.get('value') if assertion is None: print('Response did not contain a valid SAML assertion!') return 1 awsroles = [] root = ET.fromstring(base64.b64decode(assertion)) for saml2attribute in root.iter('{urn:oasis:names:tc:SAML:2.0:assertion}Attribute'): if (saml2attribute.get('Name') == 'https://aws.amazon.com/SAML/Attributes/Role'): for saml2attributevalue in saml2attribute.iter('{urn:oasis:names:tc:SAML:2.0:assertion}AttributeValue'): awsroles.append(saml2attributevalue.text) for awsrole in awsroles: chunks = awsrole.split(',') if'saml-provider' in chunks[0]: newawsrole = chunks[1] + ',' + chunks[0] index = awsroles.index(awsrole) awsroles.insert(index, newawsrole) awsroles.remove(awsrole) if len(awsroles) > 1: i = 0 print('Please choose the role you would like to assume:') for awsrole in awsroles: print('[{}]: '.format(awsrole.split(',')[0])) i += 1 selectedroleindex = raw_input() print('Selection: {}'.format(selectedroleindex)) # Basic sanity check of input if int(selectedroleindex) > (len(awsroles) - 1): print('You selected an invalid role index, please try again') return 1 role_arn = awsroles[int(selectedroleindex)].split(',')[0] principal_arn = awsroles[int(selectedroleindex)].split(',')[1] else: role_arn = awsroles[0].split(',')[0] principal_arn = awsroles[0].split(',')[1] sts = botocore.session.create_client('sts') token = sts.assume_role_with_saml(role_arn, principal_arn, assertion) credentials = {} credentials['aws_access_key_id'] = token.credentials.access_key credentials['aws_secret_access_key'] = token.credentials.secret_key credentials['aws_session_token'] = token.credentials.session_token return credentials
def saml_login(profile, region, url, user, password=None, role=None, print_env_vars=False, overwrite_default_credentials=False): session = requests.Session() response = session.get(url) keyring_key = 'aws-minion.saml' password = password or keyring.get_password(keyring_key, user) if not password: password = click.prompt('Password', hide_input=True) with Action('Authenticating against {url}..', **vars()) as act: # NOTE: parameters are hardcoded for Shibboleth IDP data = {'j_username': user, 'j_password': password, 'submit': 'Login'} response2 = session.post(response.url, data=data) saml_xml = get_saml_response(response2.text) if not saml_xml: act.error('LOGIN FAILED') click.secho( 'SAML login with user "{}" failed, please check your username and password.\n' .format(user) + 'You might need to change the password in your keyring (e.g. Mac OS X keychain) ' + 'or use the "--password" option.', bold=True, fg='blue') return url = get_form_action(response2.text) encoded_xml = codecs.encode(saml_xml.encode('utf-8'), 'base64') response3 = session.post(url, data={'SAMLResponse': encoded_xml}) account_names = get_account_names(response3.text) keyring.set_password(keyring_key, user, password) with Action('Checking SAML roles..') as act: roles = get_roles(saml_xml) if not roles: act.error('NO VALID ROLE FOUND') return if len(roles) == 1: provider_arn, role_arn = roles[0] elif role: matching_roles = [ _role for _role in roles if role in get_role_label(_role, account_names) ] if not matching_roles or len(matching_roles) > 1: raise click.UsageError( 'Given role (--role) was not found or not unique') provider_arn, role_arn = matching_roles[0] else: roles.sort() provider_arn, role_arn = choice( 'Multiple roles found, please select one.', [(r, get_role_label(r, account_names)) for r in roles]) with Action('Assuming role "{role_label}"..', role_label=get_role_label((provider_arn, role_arn), account_names)): saml_assertion = codecs.encode(saml_xml.encode('utf-8'), 'base64').decode('ascii').replace( '\n', '') # botocore NEEDS some credentials, but does not care about their actual values os.environ['AWS_ACCESS_KEY_ID'] = 'fake123' os.environ['AWS_SECRET_ACCESS_KEY'] = 'fake123' try: session = botocore.session.get_session() sts = session.get_service('sts') operation = sts.get_operation('AssumeRoleWithSAML') endpoint = sts.get_endpoint(region) endpoint._signature_version = None http_response, response_data = operation.call( endpoint, role_arn=role_arn, principal_arn=provider_arn, SAMLAssertion=saml_assertion) finally: del os.environ['AWS_ACCESS_KEY_ID'] del os.environ['AWS_SECRET_ACCESS_KEY'] key_id = response_data['Credentials']['AccessKeyId'] secret = response_data['Credentials']['SecretAccessKey'] session_token = response_data['Credentials']['SessionToken'] if print_env_vars: # different AWS SDKs expect either AWS_SESSION_TOKEN or AWS_SECURITY_TOKEN, so set both click.secho(dedent('''\ # environment variables with temporary AWS credentials: export AWS_ACCESS_KEY_ID="{key_id}" export AWS_SECRET_ACCESS_KEY="{secret}" export AWS_SESSION_TOKEN="{session_token}") export AWS_SECURITY_TOKEN="{session_token}"''').format(**vars()), fg='blue') profiles_to_write = set([profile]) if overwrite_default_credentials: profiles_to_write.add('default') with Action('Writing temporary AWS credentials..'): for prof in profiles_to_write: write_aws_credentials(prof, key_id, secret, session_token)
def saml_login(profile, region, url, user, password=None, role=None, print_env_vars=False, overwrite_default_credentials=False): session = requests.Session() response = session.get(url) keyring_key = 'aws-minion.saml' password = password or keyring.get_password(keyring_key, user) if not password: password = click.prompt('Password', hide_input=True) with Action('Authenticating against {url}..', **vars()) as act: # NOTE: parameters are hardcoded for Shibboleth IDP data = {'j_username': user, 'j_password': password, 'submit': 'Login'} response2 = session.post(response.url, data=data) saml_xml = get_saml_response(response2.text) if not saml_xml: act.error('LOGIN FAILED') click.secho('SAML login with user "{}" failed, please check your username and password.\n'.format(user) + 'You might need to change the password in your keyring (e.g. Mac OS X keychain) ' + 'or use the "--password" option.', bold=True, fg='blue') return url = get_form_action(response2.text) encoded_xml = codecs.encode(saml_xml.encode('utf-8'), 'base64') response3 = session.post(url, data={'SAMLResponse': encoded_xml}) account_names = get_account_names(response3.text) keyring.set_password(keyring_key, user, password) with Action('Checking SAML roles..') as act: roles = get_roles(saml_xml) if not roles: act.error('NO VALID ROLE FOUND') return if len(roles) == 1: provider_arn, role_arn = roles[0] elif role: matching_roles = [_role for _role in roles if role in get_role_label(_role, account_names)] if not matching_roles or len(matching_roles) > 1: raise click.UsageError('Given role (--role) was not found or not unique') provider_arn, role_arn = matching_roles[0] else: roles.sort() provider_arn, role_arn = choice('Multiple roles found, please select one.', [(r, get_role_label(r, account_names)) for r in roles]) with Action('Assuming role "{role_label}"..', role_label=get_role_label((provider_arn, role_arn), account_names)): saml_assertion = codecs.encode(saml_xml.encode('utf-8'), 'base64').decode('ascii').replace('\n', '') # botocore NEEDS some credentials, but does not care about their actual values os.environ['AWS_ACCESS_KEY_ID'] = 'fake123' os.environ['AWS_SECRET_ACCESS_KEY'] = 'fake123' try: session = botocore.session.get_session() sts = session.get_service('sts') operation = sts.get_operation('AssumeRoleWithSAML') endpoint = sts.get_endpoint(region) endpoint._signature_version = None http_response, response_data = operation.call(endpoint, role_arn=role_arn, principal_arn=provider_arn, SAMLAssertion=saml_assertion) finally: del os.environ['AWS_ACCESS_KEY_ID'] del os.environ['AWS_SECRET_ACCESS_KEY'] key_id = response_data['Credentials']['AccessKeyId'] secret = response_data['Credentials']['SecretAccessKey'] session_token = response_data['Credentials']['SessionToken'] if print_env_vars: # different AWS SDKs expect either AWS_SESSION_TOKEN or AWS_SECURITY_TOKEN, so set both click.secho(dedent('''\ # environment variables with temporary AWS credentials: export AWS_ACCESS_KEY_ID="{key_id}" export AWS_SECRET_ACCESS_KEY="{secret}" export AWS_SESSION_TOKEN="{session_token}") export AWS_SECURITY_TOKEN="{session_token}"''').format(**vars()), fg='blue') profiles_to_write = set([profile]) if overwrite_default_credentials: profiles_to_write.add('default') with Action('Writing temporary AWS credentials..'): for prof in profiles_to_write: write_aws_credentials(prof, key_id, secret, session_token)