def _get_auth_from_env( env: Mapping[str, str]) -> Optional[requests_aws4auth.AWS4Auth]: aws_in_docker = env.get('AWS_CONTAINER_CREDENTIALS_RELATIVE_URI') if aws_in_docker: # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html response = requests.get(f'http://169.254.170.2{aws_in_docker}') response.raise_for_status() credentials = response.json() access_key_id = credentials.get('AccessKeyId') secret_access_key = credentials.get('SecretAccessKey') session_token = credentials.get('Token') else: session = boto3.Session() credentials = session.get_credentials() if not credentials: return None access_key_id = credentials.access_key secret_access_key = credentials.secret_key session_token = credentials.token if not access_key_id: return None region = env.get('AWS_REGION', 'eu-west-3') return requests_aws4auth.AWS4Auth(access_key_id, secret_access_key, region, 'es', session_token=session_token)
def create_elasticsearch_client(self): """Build an elasticsearch client using the various parameters passed into this task.""" if self.connection_type == 'aws': service = 'es' region = 'us-east-1' credentials = boto3.Session().get_credentials() awsauth = requests_aws4auth.AWS4Auth(region=region, service=service, refreshable_credentials=credentials) return elasticsearch.Elasticsearch( hosts=[{'host': self.host, 'port': 443}], http_auth=awsauth, use_ssl=True, verify_certs=True, timeout=self.timeout, retry_on_status=(HTTP_CONNECT_TIMEOUT_STATUS_CODE, HTTP_GATEWAY_TIMEOUT_STATUS_CODE), retry_on_timeout=True, connection_class=RequestsHttpConnection, serializer=JSONSerializerPython2(), ) else: return elasticsearch.Elasticsearch( hosts=self.host, timeout=self.timeout, retry_on_status=(HTTP_CONNECT_TIMEOUT_STATUS_CODE, HTTP_GATEWAY_TIMEOUT_STATUS_CODE), retry_on_timeout=True, )
def includeme(config): p = urllib.parse.urlparse(config.registry.settings["elasticsearch.url"]) qs = urllib.parse.parse_qs(p.query) kwargs = { "hosts": [urllib.parse.urlunparse(p[:2] + ("", ) * 4)], "verify_certs": True, "ca_certs": certifi.where(), "timeout": 2, "retry_on_timeout": False, "serializer": serializer.serializer, "max_retries": 1, } aws_auth = bool(qs.get("aws_auth", False)) if aws_auth: aws_region = qs.get("region", ["us-east-1"])[0] kwargs["connection_class"] = elasticsearch.RequestsHttpConnection kwargs["http_auth"] = requests_aws4auth.AWS4Auth( config.registry.settings["aws.key_id"], config.registry.settings["aws.secret_key"], aws_region, "es", ) config.registry["elasticsearch.client"] = elasticsearch.Elasticsearch( **kwargs) config.registry["elasticsearch.index"] = p.path.strip("/") config.registry["elasticsearch.shards"] = int(qs.get("shards", ["1"])[0]) config.registry["elasticsearch.replicas"] = int( qs.get("replicas", ["0"])[0]) config.add_request_method(es, name="es", reify=True) from warehouse.search.tasks import reindex config.add_periodic_task(crontab(minute=0, hour=6), reindex)
def handling_errors(): auth = requests_aws4auth.AWS4Auth('<ID>', '', 'eu-west-1', '') r = requests.get('http://s3.eu-west-1.amazonaws.com', auth=auth) root = ET.fromstring(r.text) for element in root: print('Tag: ' + element.tag) for element in root.findall('Message'): print(element.tag + ': ' + element.text)
def awsauth(self): if not hasattr(self, '_awsauth'): session = boto3.Session() credentials = session.get_credentials().get_frozen_credentials() self._awsauth = requests_aws4auth.AWS4Auth( credentials.access_key, credentials.secret_key, session.region_name, self.service, session_token=credentials.token, ) return self._awsauth
def _aws_auth(): # https://github.com/sam-washington/requests-aws4auth/pull/2 session = boto3.Session() print("Retrieving credentials (profile_name={}, region_name={})".format(session.profile_name, session.region_name), file=sys.stderr) credentials = session.get_credentials() aws4auth = requests_aws4auth.AWS4Auth(credentials.access_key, credentials.secret_key, session.region_name, "es", session_token=credentials.token) def wrapped_aws4auth(request): return aws4auth(request) return wrapped_aws4auth
def _aws_auth() -> Any: credentials = session.get_credentials() aws4auth = requests_aws4auth.AWS4Auth( credentials.access_key, credentials.secret_key, session.region_name, "es", session_token=credentials.token, ) def wrapped_aws4auth(request: Any) -> Any: return aws4auth(request) return wrapped_aws4auth
def aws_request(url, data, operation): a = requests_aws4auth.AWS4Auth(*read_aws_credentials(), "us-west-2", "execute-api") req = requests.Request('POST', url=url, json=data, auth=a, headers={ "Content-Type": "application/json", "X-Amz-Target": operation }) prepared = req.prepare() r = requests.Session().send(prepared) return json.dumps(r.json())
def send(self, query, variables): aws_session = boto3.session.Session() creds = aws_session.get_credentials().get_frozen_credentials() auth = requests_aws4auth.AWS4Auth( creds.access_key, creds.secret_key, aws_session.region_name, self.service_name, session_token=creds.token, ) transport = gql.transport.requests.RequestsHTTPTransport( url=self.appsync_graphql_url, use_json=True, headers=self.headers, auth=auth, ) resp = transport.execute(query, variables) if resp.errors: raise Exception(f'Appsync resp error: `{resp.errors}` from query `{query}`, variables `{variables}`')
def es_authentication(service, region, es_host): credentials = boto3.Session( aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY).get_credentials() aws_auth = requests_aws4auth.AWS4Auth(credentials.access_key, credentials.secret_key, region, service, session_token=credentials.token) es = elasticsearch.Elasticsearch( hosts=[{ 'host': es_host, 'port': 443 }], http_auth=aws_auth, use_ssl=True, verify_certs=True, connection_class=elasticsearch.RequestsHttpConnection, timeout=30) return es
def _get_auth_from_env( env: Mapping[str, str]) -> Optional[requests_aws4auth.AWS4Auth]: aws_in_docker = env.get('AWS_CONTAINER_CREDENTIALS_RELATIVE_URI') if aws_in_docker: # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html response = requests.get(f'http://169.254.170.2{aws_in_docker}') response.raise_for_status() credentials = response.json() access_key_id = credentials.get('AccessKeyId') secret_access_key = credentials.get('SecretAccessKey') session_token = credentials.get('Token') else: access_key_id = env.get('AWS_ACCESS_KEY_ID') secret_access_key = env.get('AWS_SECRET_ACCESS_KEY') session_token = None if not access_key_id: return None return requests_aws4auth.AWS4Auth(access_key_id, secret_access_key, _find_aws_region(env), 'es', session_token=session_token)
def reindex(self, request): """ Recreate the Search Index. """ r = redis.StrictRedis.from_url(request.registry.settings["celery.scheduler_url"]) try: with SearchLock(r, timeout=30 * 60, blocking_timeout=30): p = urllib.parse.urlparse(request.registry.settings["elasticsearch.url"]) qs = urllib.parse.parse_qs(p.query) kwargs = { "hosts": [urllib.parse.urlunparse(p[:2] + ("",) * 4)], "verify_certs": True, "ca_certs": certifi.where(), "timeout": 30, "retry_on_timeout": True, "serializer": serializer.serializer, } aws_auth = bool(qs.get("aws_auth", False)) if aws_auth: aws_region = qs.get("region", ["us-east-1"])[0] kwargs["connection_class"] = elasticsearch.RequestsHttpConnection kwargs["http_auth"] = requests_aws4auth.AWS4Auth( request.registry.settings["aws.key_id"], request.registry.settings["aws.secret_key"], aws_region, "es", ) client = elasticsearch.Elasticsearch(**kwargs) number_of_replicas = request.registry.get("elasticsearch.replicas", 0) refresh_interval = request.registry.get("elasticsearch.interval", "1s") # We use a randomly named index so that we can do a zero downtime reindex. # Essentially we'll use a randomly named index which we will use until all # of the data has been reindexed, at which point we'll point an alias at # our randomly named index, and then delete the old randomly named index. # Create the new index and associate all of our doc types with it. index_base = request.registry["elasticsearch.index"] random_token = binascii.hexlify(os.urandom(5)).decode("ascii") new_index_name = "{}-{}".format(index_base, random_token) doc_types = request.registry.get("search.doc_types", set()) shards = request.registry.get("elasticsearch.shards", 1) # Create the new index with zero replicas and index refreshes disabled # while we are bulk indexing. new_index = get_index( new_index_name, doc_types, using=client, shards=shards, replicas=0, interval="-1", ) new_index.create(wait_for_active_shards=shards) # From this point on, if any error occurs, we want to be able to delete our # in progress index. try: request.db.execute("SET statement_timeout = '600s'") for _ in parallel_bulk( client, _project_docs(request.db), index=new_index_name ): pass except: # noqa new_index.delete() raise finally: request.db.rollback() request.db.close() # Now that we've finished indexing all of our data we can update the # replicas and refresh intervals. client.indices.put_settings( index=new_index_name, body={ "index": { "number_of_replicas": number_of_replicas, "refresh_interval": refresh_interval, } }, ) # Point the alias at our new randomly named index and delete the old index. if client.indices.exists_alias(name=index_base): to_delete = set() actions = [] for name in client.indices.get_alias(name=index_base): to_delete.add(name) actions.append({"remove": {"index": name, "alias": index_base}}) actions.append({"add": {"index": new_index_name, "alias": index_base}}) client.indices.update_aliases({"actions": actions}) client.indices.delete(",".join(to_delete)) else: client.indices.put_alias(name=index_base, index=new_index_name) except redis.exceptions.LockError as exc: raise self.retry(countdown=60, exc=exc)
import sys import requests import requests_aws4auth as aws4auth import xml.etree.ElementTree as ET import xml.dom.minidom as minidom access_id = '' access_key = '' region = 'eu-west-2' endpoint = 's3.{}.amazonaws.com'.format(region) auth = aws4auth.AWS4Auth(access_id, access_key, region, 's3') def xml_pprint(xml_string): print(minidom.parseString(xml_string).toprettyxml()) def download_file(bucket, s3_name): url = 'http://{}/{}/{}'.format(endpoint, bucket, s3_name) print('download file '+url) response = requests.get(url, auth=auth) print(response) if response.ok: open(s3_name, 'wb').write(response.content) print('Downloaded {} OK'.format(s3_name)) else: xml_pprint(response.text) def upload_file(bucket, local_path): data = open(local_path, 'rb').read() url = 'http://{}/{}/{}'.format(endpoint, bucket, local_path) print('upload file '+url)
#!/usr/bin/env python3 import sys import requests import requests_aws4auth import xml.etree.ElementTree as ET import xml.dom.minidom as minidom import mimetypes access_id = '' access_key = '' region = 'US East (N. Virginia)' endpoint = 's3-{}.amazonaws.com'.format(region) auth = requests_aws4auth.AWS4Auth(access_id, access_key, region, 's3') ns = 'http://s3.amazonaws.com/doc/2006-03-01/' def xml_pprint(xml_string): print(minidom.parseString(xml_string).toprettyxml()) def create_bucket(bucket): XML = ET.Element('CreateBucketConfiguration') XML.attrib['xmlns'] = ns location = ET.SubElement(XML, 'LocationConstraint') location.text = auth.region data = ET.tostring(XML, encoding='utf-8') url = 'http://{}.{}'.format(bucket, endpoint) r = requests.put(url, data=data, auth=auth) if r.ok: print('Created bucket {} OK'.format(bucket))
def __service_request(self, method, body=None, params=None): '''Submits a network requests and waits for a response. Arguments: method - the functin to be called to execute the request. body - the json content submited with the request params - query string parameters sent with the request Returns: The json content from response wrapped by a Data object. The returned object will have a PATH attribute that is this path object and a RESPONSE attribute that is the response object for the http request. The Data object will be read only. ''' if self.__config.session: if self.__config.role_arn: sts = self.__config.session.client('sts') res = sts.assume_role( RoleArn=self.__config.role_arn, RoleSessionName=self.__config.role_session_name or 'cgf_service_client') access_key = res['Credentials']['AccessKeyId'] secret_key = res['Credentials']['SecretAccessKey'] session_token = res['Credentials']['SessionToken'] auth_description = self.__config.role_arn else: session_credentials = self.__config.session.get_credentials( ).get_frozen_credentials() access_key = session_credentials.access_key secret_key = session_credentials.secret_key session_token = session_credentials.token auth_description = 'session' auth = requests_aws4auth.AWS4Auth( access_key, secret_key, self.__config.session.region_name, 'execute-api', session_token=session_token) else: auth = None auth_description = None if self.__config.verbose: print('HTTP', method.__name__.upper(), self.__url, params, body, auth_description) response = method(self.__url, params=params, json=body, auth=auth) if self.__config.verbose: print(' -->', response.status_code, response.text) error.HttpError.for_response(response) if response.content: result = response.json() # TODO: the generated api gateway lambda displatch mapping injects this 'result' thing # in there, it shouldn't do this because it breaks the contract defined by the swagger. # To fix it, we need to: # # 1) change swagger_processor.lambda_dispatch to return the response provided by the # lambda directly. Problem is that breaks the additional_response_template_content feature, # which can probabally be removed all together. # # 2) change the c++ client generator handle the missing result property # # 3) change cgp to handle the missing result property # # 4) remove the following code # result = result.get('result', None) if result is None: return RuntimeError( 'Service response did not include the stupid result property.' ) # # END TODO return Data(result, read_only=True, PATH=self, RESPONSE=response) else: return None
import sys import requests import requests_aws4auth as AA import xml.etree.ElementTree as ET import xml.dom.minidom as minidom import mimetypes f = open('ID.txt') #아이디 받아오기 access_id = f.read() f = open('Key.txt') #엑세스 키 받아오기 access_key = f.read() region = 'ap-northeast-2' #위치 설정 endpoint = 's3-{}.amazonaws.com'.format(region) #s3를 사용하는데 알맞은 Url형태로 만듦ㅎ auth = AA.AWS4Auth(access_id, access_key, region, 's3') #s3에 접속 ns = 'http://s3.amazonaws.com/doc/2006-03-01/' #????? def xml_pprint(element): print(minidom.parseString( element).toprettyxml()) #엘레먼트 트리의 xml형태의 변수를 xml형태에 맞춰 출력시켜준다. def upload_file(bucket, s3_name, local_path, acl='private'): data = open(local_path, 'rb').read() #모르겠다. url = 'http://{}.{}/{}'.format(bucket, endpoint, s3_name) #준비해둔 url 양식을 끼워넣음 headers = {'x-amz-acl': acl} #헤더중에 접근권한 성정 변경 mimetype = mimetypes.guess_type(local_path)[0] #자료형태를 추측?? #다운로드 없이 바로 이미지 띄움 if mimetype: