Example #1
0
def _elasticsearch_connect():
    """
    Connect to configured Elasticsearch domain.

    :return: An Elasticsearch connection object.
    """

    es_url = config("ELASTICSEARCH_URL", default="localhost")
    es_port = config("ELASTICSEARCH_PORT", default=9200, cast=int)
    es_aws_region = config("ELASTICSEARCH_AWS_REGION", default="us-east-1")

    auth = AWSRequestsAuth(
        aws_access_key=settings.AWS_ACCESS_KEY_ID,
        aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
        aws_host=es_url,
        aws_region=es_aws_region,
        aws_service="es",
    )
    auth.encode = lambda x: bytes(x.encode("utf-8"))
    _es = Elasticsearch(
        host=es_url,
        port=es_port,
        connection_class=RequestsHttpConnection,
        timeout=10,
        max_retries=1,
        retry_on_timeout=True,
        http_auth=auth,
        wait_for_status="yellow",
    )
    _es.info()
    return _es
Example #2
0
 def __init__(self,
              es_host,
              region='us-east-1',
              is_lambda=True,
              index='iocs'):
     AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
     AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
     if is_lambda:
         aws_token = os.environ["AWS_SESSION_TOKEN"]
         self.auth = AWSRequestsAuth(
             aws_access_key=AWS_ACCESS_KEY_ID,
             aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
             aws_token=aws_token,
             aws_host=es_host,
             aws_region=region,
             aws_service='es')
     else:
         self.auth = AWSRequestsAuth(
             aws_access_key=AWS_ACCESS_KEY_ID,
             aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
             aws_host=es_host,
             aws_region=region,
             aws_service='es')
     self.session = requests.session()
     self.session.auth = self.auth
     self.es_host = es_host
    def __call__(self, r):
        try:
            # Host used in signature *MUST* always match with Host HTTP header.
            host = r.headers.get('Host')
            if not host:
                _, _, host, _, _, _, _ = parse_url(r.url)
                r.headers['Host'] = host
            if self.domain is not None:
                aws_params = self._parse_url(self.domain)
            else:
                aws_params = self._parse_url(host)
        except ValueError:
            aws_params = {"region": "us-west-2", "service": "svs"}
        except Exception as error:
            aws_params = {"region": "us-west-2", "service": "svs"}

        aws_request = AWSRequestsAuth(
            aws_access_key=self.aws_access_key,
            aws_secret_access_key=self.aws_secret_access_key,
            aws_host=host,
            aws_region=aws_params['region'],
            aws_service=aws_params['service'],
            aws_token=self.aws_token)

        return aws_request.__call__(r)
Example #4
0
def _elasticsearch_connect() -> Elasticsearch:
    """
    Connect to an Elasticsearch indices at the configured domain. This method also
    handles AWS authentication using the AWS access key ID and the secret access key.

    :return: an Elasticsearch client
    """

    log.info(
        f"Connecting to {ELASTICSEARCH_URL}:{ELASTICSEARCH_PORT} with AWS auth"
    )
    auth = AWSRequestsAuth(
        aws_access_key=AWS_ACCESS_KEY_ID,
        aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
        aws_host=ELASTICSEARCH_URL,
        aws_region=AWS_REGION,
        aws_service="es",
    )
    auth.encode = lambda x: bytes(x.encode("utf-8"))
    es = Elasticsearch(
        host=ELASTICSEARCH_URL,
        port=ELASTICSEARCH_PORT,
        connection_class=RequestsHttpConnection,
        http_auth=auth,
        timeout=TWELVE_HOURS_SEC,
    )
    es.info()
    return es
Example #5
0
def _elasticsearch_connect(timeout=300):
    """
    Connect to configured Elasticsearch domain.

    :param timeout: How long to wait before ANY request to Elasticsearch times
    out. Because we use parallel bulk uploads (which sometimes wait long periods
    of time before beginning execution), a value of at least 30 seconds is
    recommended.
    :return: An Elasticsearch connection object.
    """

    log.info('Connecting to %s %s with AWS auth', ELASTICSEARCH_URL,
             ELASTICSEARCH_PORT)
    auth = AWSRequestsAuth(aws_access_key=AWS_ACCESS_KEY_ID,
                           aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
                           aws_host=ELASTICSEARCH_URL,
                           aws_region=AWS_REGION,
                           aws_service='es')
    auth.encode = lambda x: bytes(x.encode('utf-8'))
    es = Elasticsearch(host=ELASTICSEARCH_URL,
                       port=ELASTICSEARCH_PORT,
                       connection_class=RequestsHttpConnection,
                       timeout=timeout,
                       max_retries=10,
                       retry_on_timeout=True,
                       http_auth=auth,
                       wait_for_status='yellow')
    es.info()
    return es
    def __call__(self, r):
        try:
            # Host used in signature *MUST* always match with Host HTTP header.
            host = r.headers.get('Host')
            if not host:
                _, _, host, _, _, _, _ = parse_url(r.url)
                r.headers['Host'] = host
            if self.domain is not None:
                aws_params = self._parse_url(self.domain)
            else:
                aws_params = self._parse_url(host)
        except ValueError:
            print("ERROR: Could not parse neccessary information from URL.")
            raise
        except Exception as error:
            print("Error parsing URL: %s" % error)
            raise

        if isinstance(r.body, io.RawIOBase):
            r.body = r.body.readall()
        elif isinstance(r.body, (io.BufferedIOBase, io.TextIOBase)):
            r.body = r.body.read()

        aws_request = AWSRequestsAuth(
            aws_access_key=self.aws_access_key,
            aws_secret_access_key=self.aws_secret_access_key,
            aws_host=host,
            aws_region=aws_params['region'],
            aws_service=aws_params['service'],
            aws_token=self.aws_token)

        return aws_request.__call__(r)
    def __call__(self, r: requests.PreparedRequest) -> requests.PreparedRequest:
        if self.access_key is None:
            fatal_plugin_error(f"ERROR: Missing {AWS_ACCESS_KEY} in environment")
        if self.secret_key is None:
            fatal_plugin_error(f"ERROR: Missing {AWS_SECRET_KEY} in environment")

        try:
            host = (
                r.headers.get("Host").decode("utf-8")
                if "Host" in r.headers
                else urlparse(r.url).netloc
            )

            if self.region is None or self.service is None:
                p = parse(
                    r"(?P<region>[^\.]+)\.(?P<service>.+)\.api\.nifcloud\.com", host
                )
                self.region = p["region"]
                self.service = p["service"]

        except Exception as error:
            fatal_plugin_error(f"ERROR: {error}")
            raise

        auth = AWSRequestsAuth(
            aws_access_key=self.access_key,
            aws_secret_access_key=self.secret_key,
            aws_host=host,
            aws_region=self.region,
            aws_service=self.service,
        )

        return auth.__call__(r)
def _elasticsearch_connect():
    """
    Connect to configured Elasticsearch domain.

    :return: An Elasticsearch connection object.
    """
    auth = AWSRequestsAuth(
        aws_access_key=settings.AWS_ACCESS_KEY_ID,
        aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
        aws_host=settings.ELASTICSEARCH_URL,
        aws_region=settings.ELASTICSEARCH_AWS_REGION,
        aws_service='es'
    )
    auth.encode = lambda x: bytes(x.encode('utf-8'))
    _es = Elasticsearch(
        host=settings.ELASTICSEARCH_URL,
        port=settings.ELASTICSEARCH_PORT,
        connection_class=RequestsHttpConnection,
        timeout=10,
        max_retries=99,
        retry_on_timeout=True,
        http_auth=auth,
        wait_for_status='yellow'
    )
    _es.info()
    return _es
 def test_path_with_querystring(self):
     """
     Assert we generate the 'correct' cannonical query string
     and path for request that includes a query stirng
     """
     url = 'http://search-foo.us-east-1.es.amazonaws.com:80/my_index/?pretty=True'
     mock_request = mock.Mock()
     mock_request.url = url
     self.assertEqual('/my_index/', AWSRequestsAuth.get_caononical_path(mock_request))
     self.assertEqual('pretty=True', AWSRequestsAuth.get_canonical_querystring(mock_request))
 def test_characters_escaped_in_path(self):
     """
     Assert we generate the 'correct' cannonical query string
     and path a request with characters that need to be escaped
     """
     url = 'http://search-foo.us-east-1.es.amazonaws.com:80/+foo.*/_stats'
     mock_request = mock.Mock()
     mock_request.url = url
     self.assertEqual('/%2Bfoo.%2A/_stats', AWSRequestsAuth.get_caononical_path(mock_request))
     self.assertEqual('', AWSRequestsAuth.get_canonical_querystring(mock_request))
Example #11
0
 def test_characters_escaped_in_path(self):
     """
     Assert we generate the 'correct' cannonical query string
     and path a request with characters that need to be escaped
     """
     url = 'http://search-foo.us-east-1.es.amazonaws.com:80/+foo.*/_stats'
     mock_request = mock.Mock()
     mock_request.url = url
     self.assertEqual('/%2Bfoo.%2A/_stats',
                      AWSRequestsAuth.get_canonical_path(mock_request))
     self.assertEqual(
         '', AWSRequestsAuth.get_canonical_querystring(mock_request))
    def test_no_query_params(self):
        """
        Assert we generate the 'correct' cannonical query string
        and canonical path for a request with no query params

        Correct is relative here b/c 'correct' simply means what
        the AWS Elasticsearch service expects
        """
        url = 'http://search-foo.us-east-1.es.amazonaws.com:80/'
        mock_request = mock.Mock()
        mock_request.url = url
        self.assertEqual('/', AWSRequestsAuth.get_caononical_path(mock_request))
        self.assertEqual('', AWSRequestsAuth.get_canonical_querystring(mock_request))
Example #13
0
 def test_path_with_querystring(self):
     """
     Assert we generate the 'correct' cannonical query string
     and path for request that includes a query stirng
     """
     url = 'http://search-foo.us-east-1.es.amazonaws.com:80/my_index/?pretty=True'
     mock_request = mock.Mock()
     mock_request.url = url
     self.assertEqual('/my_index/',
                      AWSRequestsAuth.get_canonical_path(mock_request))
     self.assertEqual(
         'pretty=True',
         AWSRequestsAuth.get_canonical_querystring(mock_request))
Example #14
0
    def test_no_query_params(self):
        """
        Assert we generate the 'correct' cannonical query string
        and canonical path for a request with no query params

        Correct is relative here b/c 'correct' simply means what
        the AWS Elasticsearch service expects
        """
        url = 'http://search-foo.us-east-1.es.amazonaws.com:80/'
        mock_request = mock.Mock()
        mock_request.url = url
        self.assertEqual('/', AWSRequestsAuth.get_canonical_path(mock_request))
        self.assertEqual(
            '', AWSRequestsAuth.get_canonical_querystring(mock_request))
Example #15
0
    def test_auth_for_post(self):
        auth = AWSRequestsAuth(
            aws_access_key='YOURKEY',
            aws_secret_access_key='YOURSECRET',
            aws_host='search-foo.us-east-1.es.amazonaws.com',
            aws_region='us-east-1',
            aws_service='es')
        url = 'http://search-foo.us-east-1.es.amazonaws.com:80/'
        mock_request = mock.Mock()
        mock_request.url = url
        mock_request.method = "POST"
        mock_request.body = b'foo=bar'
        mock_request.headers = {
            'Content-Type': 'application/x-www-form-urlencoded',
        }

        frozen_datetime = datetime.datetime(2016, 6, 18, 22, 4, 5)
        with mock.patch('datetime.datetime') as mock_datetime:
            mock_datetime.utcnow.return_value = frozen_datetime
            auth(mock_request)
        self.assertEqual(
            {
                'Authorization':
                'AWS4-HMAC-SHA256 Credential=YOURKEY/20160618/us-east-1/es/aws4_request, '
                'SignedHeaders=host;x-amz-date, '
                'Signature=a6fd88e5f5c43e005482894001d9b05b43f6710e96be6098bcfcfccdeb8ed812',
                'Content-Type':
                'application/x-www-form-urlencoded',
                'x-amz-date':
                '20160618T220405Z',
            }, mock_request.headers)
Example #16
0
    def test_auth_for_post_with_unicode_body_python3(self):
        auth = AWSRequestsAuth(
            aws_access_key='YOURKEY',
            aws_secret_access_key='YOURSECRET',
            aws_host='search-foo.us-east-1.es.amazonaws.com',
            aws_region='us-east-1',
            aws_service='es')
        url = 'http://search-foo.us-east-1.es.amazonaws.com:80/'
        mock_request = mock.Mock()
        mock_request.url = url
        mock_request.method = "POST"
        mock_request.body = 'foo=bar\xc3'
        mock_request.headers = {
            'Content-Type': 'application/x-www-form-urlencoded',
        }

        frozen_datetime = datetime.datetime(2016, 6, 18, 22, 4, 5)
        with mock.patch('datetime.datetime') as mock_datetime:
            mock_datetime.utcnow.return_value = frozen_datetime
            auth(mock_request)

        self.assertEqual(
            {
                'Authorization':
                'AWS4-HMAC-SHA256 Credential=YOURKEY/20160618/us-east-1/es/aws4_request, '
                'SignedHeaders=host;x-amz-date, '
                'Signature=0836dae4bce95c1bcdbd3751c84c0c7e589ba7c81331bab92d0e1acb94adcdd9',
                'Content-Type':
                'application/x-www-form-urlencoded',
                'x-amz-date':
                '20160618T220405Z',
            }, mock_request.headers)
Example #17
0
    def login(self, username, password):
        """
        Retrieve the aws credentials used by the apis.

        The login is staged, p1 we login to gigya, p2 we use p1 credentials
        in irbt login api, that finally return aws credentials to
        use in all the subsequent calls (mqtt and api gateway)
        """
        # discover cloud settings
        self._disc()

        # login to gigya (get uid, timestamp and signature)
        gigya_login = self._gigya_login(username, password)

        # get aws keys through irbt login api
        try:
            (self.access_key_id,
             self.secret_key,
             self.session_token) = self._irbt_login_api(
                gigya_login['UIDSignature'],
                gigya_login['signatureTimestamp'],
                gigya_login['UID'])
        except KeyError:
            logger.error('Authentication failed, wrong login/password')
            return -1
        # use AWSRequestsAuth module to set the aws authentication headers
        # (Signature Version 4 Signing Process)
        self.auth = AWSRequestsAuth(
            aws_access_key=self.access_key_id,
            aws_secret_access_key=self.secret_key,
            aws_token=self.session_token,
            aws_host=self._aws_host,
            aws_region=self._aws_region,
            aws_service='execute-api')
        return True
Example #18
0
def handler(event, context):
    # For this function, we don't care about 'event' and 'context',
    # but they need to be in the function signature anyway.

    esinfo = {}
    esendpoint = 'vpc-shop-v5-f4mgyu2gin24qdpouzn3icyz7m.eu-west-1.es.amazonaws.com'

    session = boto3.session.Session()
    credentials = session.get_credentials().get_frozen_credentials()
    awsauth = AWSRequestsAuth(
            aws_access_key=credentials.access_key,
            aws_secret_access_key=credentials.secret_key,
            aws_token=credentials.token,
            aws_host=esendpoint,
            aws_region=session.region_name,
            aws_service='es'
        )

    es = Elasticsearch(
            hosts=[{'host': esendpoint, 'port': 443}],
            http_auth=awsauth,
            use_ssl=True,
            verify_certs=True,
            ca_certs=certifi.where(),
            connection_class=RequestsHttpConnection
        )
    esinfo = es.info();

    print(esinfo)
Example #19
0
    def __init__(self):
        self.session = requests.Session()
        self.session.headers.update({
            'X-Cuenca-Api-Version':
            API_VERSION,
            'User-Agent':
            f'cuenca-python/{CLIENT_VERSION}',
        })

        # basic auth
        api_key = os.getenv('CUENCA_API_KEY', '')
        api_secret = os.getenv('CUENCA_API_SECRET', '')
        self.basic_auth = (api_key, api_secret)

        # IAM auth
        aws_access_key = os.getenv('AWS_ACCESS_KEY_ID', '')
        aws_secret_access_key = os.getenv('AWS_SECRET_ACCESS_KEY', '')
        aws_region = os.getenv('AWS_DEFAULT_REGION', AWS_DEFAULT_REGION)
        if aws_access_key and aws_secret_access_key:
            self.iam_auth = AWSRequestsAuth(
                aws_access_key=aws_access_key,
                aws_secret_access_key=aws_secret_access_key,
                aws_host=self.host,
                aws_region=aws_region,
                aws_service=AWS_SERVICE,
            )
Example #20
0
    def __init__(self, json_auth):
        with open(json_auth) as f:
            auth_info = json.load(f)
            self.url = auth_info['apiEndpoint']
            self.user_name = auth_info['userId']
            host = self.url.split('/')[2]
            self.navi_auth = AWSRequestsAuth(
                aws_access_key=auth_info['accessKey'],
                aws_secret_access_key=auth_info['secretKey'],
                aws_host=host,
                aws_region=auth_info['region'],
                aws_service='execute-api')

        #endpoints
        self.status_endpoint = "api/v0/status"
        self.robots_endpoint = "api/v0/users/{1}/robots"
        self.robot_endpoint = "api/v0/users/{1}/robots/{2}"
        self.sessions_endpoint = "api/v0/users/{1}/robots/{2}/sessions"
        self.session_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}"
        self.nodes_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/nodes"
        self.node_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/nodes/{4}"
        self.node_labelled_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/nodes/labelled/{4}"
        self.odo_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/odometry"
        self.big_data_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/nodes/{4}/data"
        self.big_data_element_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/nodes/{4}/data/{5}"
        self.big_data_raw_element_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/nodes/{4}/data/{5}/raw"
        self.variable_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/variables/{4}"
        self.factors_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/factors"
        self.factor_endpoint = "$factorsEndpoint/{4}"
        self.bearing_range_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/factors/bearingrange"
        self.session_ready_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/ready/{4}"
        # context
        self.sessionId = ""
        self.robotId = ""
        self.userId = ""
    def _get_auth(self) -> AWSRequestsAuth:
        """Generate an AWSRequestsAuth object using a boto session for the current/local account.

        Returns:
             AWSRequestsAuth object
        """
        if datetime.now() >= self._auth_expiration:
            session = boto3.Session()
            credentials = session.get_credentials()
            region = (
                session.region_name
                if self._neptune_endpoint.region is None
                else self._neptune_endpoint.region
            )
            auth = AWSRequestsAuth(
                aws_access_key=credentials.access_key,
                aws_secret_access_key=credentials.secret_key,
                aws_token=credentials.token,
                aws_host=f"{self._neptune_endpoint.host}:{self._neptune_endpoint.port}",
                aws_region=region,
                aws_service="neptune-db",
            )
            self._auth = auth
            self._auth_expiration = datetime.now() + timedelta(minutes=SESSION_LIFETIME_MINUTES)
        return self._auth
Example #22
0
def connectES(esEndPoint):
   print ('Connecting to the ES Endpoint {0}'.format(esEndPoint))
   try:

      access_key = os.getenv('AWS_ACCESS_KEY_ID')
      secret_access_key = os.getenv('AWS_SECRET_ACCESS_KEY')
      session_token = os.getenv('AWS_SESSION_TOKEN')
      region = os.getenv('AWS_REGION')

      auth = AWSRequestsAuth(aws_access_key=access_key,
                             aws_secret_access_key=secret_access_key,
                             aws_token=session_token,
                             aws_host=esEndPoint,
                             aws_region=region,
                             aws_service='es')

      esClient = Elasticsearch(host=esEndPoint,
                               port=443,
                               use_ssl=True,
                               connection_class=RequestsHttpConnection,
                               http_auth=auth)

      return esClient
   except Exception as E:
      print("Unable to connect to {0}".format(esEndPoint))
      print(E)
      exit(3)
Example #23
0
def get_aws_auth():
    """Get HTTP authentication for our Elastic Search AWS user"""
    return AWSRequestsAuth(aws_access_key=private.ES_ACCESS_KEY_ID,
                           aws_secret_access_key=private.ES_SECRET_ACCESS_KEY,
                           aws_host=private.ES_HOST,
                           aws_region=private.ES_REGION,
                           aws_service="es")
def amazonCall(url):
    sts_client = boto3.client('sts')
    # Call the assume_role method of the STSConnection object and pass the role
    # ARN and a role session name.
    assumedRoleObject = sts_client.assume_role(
        RoleArn="arn:aws:iam::543303400543:role/MyApiCall",
        RoleSessionName="mySession")
    # From the response that contains the assumed role, get the temporary
    # credentials that can be used to make subsequent API calls
    credentials = assumedRoleObject['Credentials']
    # Use the temporary credentials that AssumeRole returns to make a
    # connection to Amazon S3
    # s3_resource = boto3.resource(
    #     's3',
    #     aws_access_key_id = ,
    #     aws_secret_access_key = ,
    #     aws_session_token = credentials['SessionToken'],
    # )
    # print("s3 resource:",s3_resource)
    # # let's talk to our AWS Elasticsearch cluster
    auth = AWSRequestsAuth(
        aws_access_key=credentials['AccessKeyId'],
        aws_secret_access_key=credentials['SecretAccessKey'],
        aws_host='ec2-18-220-161-116.us-east-2.compute.amazonaws.com',
        aws_region='us-east-2',
        aws_service='compute')
    response = requests.get(url, auth=auth)
    return response
Example #25
0
def lambda_handler(event, context):
    print('Event Received: {}'.format(json.dumps(event)))

    bucket_name = event['Records'][0]['s3']['bucket']['name']
    # Wait for other objects to finish being uploaded
    time.sleep(30)
    resp = s3.list_objects_v2(Bucket=bucket_name)['Contents']
    all_keys = [d['Key'] for d in resp]
    final_keys = [
        key for key in all_keys
        if key.startswith('gisaid/') and key.endswith('vcf.gz')
    ]
    locations = [f's3://{bucket_name}/{d}' for d in final_keys]
    payload = {
        "id": "gisaid",
        "vcfLocations": locations,
    }
    api = BEACON_URL.split('/')[2]

    url = BEACON_URL + "/submit"
    auth = AWSRequestsAuth(
        aws_access_key=aws_access_key_id,
        aws_secret_access_key=aws_secret_access_key,
        aws_token=aws_session_token,
        aws_host=api,
        aws_region='ap-southeast-2',
        aws_service='execute-api',
    )
    response = requests.patch(url, data=json.dumps(payload), auth=auth)
    print(dir(response))
Example #26
0
def elasticsearch_client():

    kwargs = {
        'host': ES_HOST,
        'port': ES_PORT,
    }

    if ES_SSL:
        kwargs.update({
            'use_ssl': True,
            'ca_certs': certifi.where(),
        })

    if ES_SIGNING:
        kwargs.update({
            'connection_class':
            RequestsHttpConnection,
            'http_auth':
            AWSRequestsAuth(aws_host=ES_HOST,
                            aws_region=ES_REGION,
                            aws_service='es',
                            **boto_utils.get_credentials()),
        })

    return Elasticsearch(**kwargs)
Example #27
0
    def test_auth_for_get(self):
        auth = AWSRequestsAuth(
            aws_access_key='YOURKEY',
            aws_secret_access_key='YOURSECRET',
            aws_host='search-foo.us-east-1.es.amazonaws.com',
            aws_region='us-east-1',
            aws_service='es')
        url = 'http://search-foo.us-east-1.es.amazonaws.com:80/'
        mock_request = mock.Mock()
        mock_request.url = url
        mock_request.method = "GET"
        mock_request.body = None
        mock_request.headers = {}

        frozen_datetime = datetime.datetime(2016, 6, 18, 22, 4, 5)
        with mock.patch('datetime.datetime') as mock_datetime:
            mock_datetime.utcnow.return_value = frozen_datetime
            auth(mock_request)
        self.assertEqual(
            {
                'Authorization':
                'AWS4-HMAC-SHA256 Credential=YOURKEY/20160618/us-east-1/es/aws4_request, '
                'SignedHeaders=host;x-amz-date, '
                'Signature=ca0a856286efce2a4bd96a978ca6c8966057e53184776c0685169d08abd74739',
                'x-amz-date':
                '20160618T220405Z',
            }, mock_request.headers)
Example #28
0
    def _make_elastic(self):
        """create elasticsearch client"""
        elastic_host = os.environ["ES_HOST"]
        session = boto3.session.Session()
        credentials = session.get_credentials().get_frozen_credentials()
        awsauth = AWSRequestsAuth(
            # These environment variables are automatically set by Lambda
            aws_access_key=credentials.access_key,
            aws_secret_access_key=credentials.secret_key,
            aws_token=credentials.token,
            aws_host=elastic_host,
            aws_region=session.region_name,
            aws_service="es"
        )

        return Elasticsearch(
            hosts=[{"host": elastic_host, "port": 443}],
            http_auth=awsauth,
            max_backoff=get_time_remaining(self.context) if self.context else MAX_BACKOFF,
            # Give ES time to respond when under load
            timeout=ELASTIC_TIMEOUT,
            use_ssl=True,
            verify_certs=True,
            connection_class=RequestsHttpConnection
        )
Example #29
0
def get_elasticsearch_connection():
    "gets an aws-authenticated elasticsearch connection"
    session = boto3.session.Session()
    credentials = session.get_credentials().get_frozen_credentials()
    es_host = (
        "search-sciwiki-search-0-f7ntx2mpc5g6yohp6dtiwzsdiy.us-west-2.es.amazonaws.com"
    )
    awsauth = AWSRequestsAuth(
        aws_access_key=credentials.access_key,
        aws_secret_access_key=credentials.secret_key,
        aws_token=credentials.token,
        aws_host=es_host,
        aws_region=session.region_name,
        aws_service="es",
    )
    return Elasticsearch(
        hosts=[{
            "host": es_host,
            "port": 443
        }],
        http_auth=awsauth,
        use_ssl=True,
        verify_certs=True,
        connection_class=RequestsHttpConnection,
    )
Example #30
0
async def async_process_db_change(event, context):
    aws_access_key_id = os.environ['AWS_ACCESS_KEY_ID']
    aws_secret_access_key = os.environ['AWS_SECRET_ACCESS_KEY']
    aws_session_token = os.environ['AWS_SESSION_TOKEN']

    auth = AWSRequestsAuth(
        aws_access_key=aws_access_key_id,
        aws_secret_access_key=aws_secret_access_key,
        aws_token=aws_session_token,
        aws_host='0ppn9ycig5.execute-api.us-east-1.amazonaws.com',
        aws_region='us-east-1',
        aws_service='execute-api')
    latest_record = {}
    for i, record in enumerate(event['Records']):
        latest_record[record['dynamodb']['Keys']['id']['S']] = i

    all_tasks = []
    for game_id, ind in latest_record.items():
        if 'NewImage' not in event['Records'][ind]['dynamodb']:
            continue
        if not event['Records'][ind]['dynamodb']['NewImage']['id'][
                'S'].startswith('game'):
            continue
        record = convert_record(event['Records'][ind]['dynamodb']['NewImage'])
        try:
            all_tasks.append(asyncio.create_task(handle_record(record, auth)))
        except ValueError as e:
            return {
                'statusCode': 409,
                'body': json.dumps('Game state not recognized')
            }

    await asyncio.gather(*all_tasks)

    return {'statusCode': 200, 'body': json.dumps('success')}
Example #31
0
    def test_auth_for_post_with_unicode_body_python2(self):
        auth = AWSRequestsAuth(
            aws_access_key='YOURKEY',
            aws_secret_access_key='YOURSECRET',
            aws_host='search-foo.us-east-1.es.amazonaws.com',
            aws_region='us-east-1',
            aws_service='es')
        url = 'http://search-foo.us-east-1.es.amazonaws.com:80/'
        mock_request = mock.Mock()
        mock_request.url = url
        mock_request.method = "POST"
        mock_request.body = 'foo=bar\xc3'
        mock_request.headers = {
            'Content-Type': 'application/x-www-form-urlencoded',
        }

        frozen_datetime = datetime.datetime(2016, 6, 18, 22, 4, 5)
        with mock.patch('datetime.datetime') as mock_datetime:
            mock_datetime.utcnow.return_value = frozen_datetime
            auth(mock_request)

        self.assertEqual(
            {
                'Authorization':
                'AWS4-HMAC-SHA256 Credential=YOURKEY/20160618/us-east-1/es/aws4_request, '
                'SignedHeaders=host;x-amz-date, '
                'Signature=88046be72423b267de5e7e604aaffb2c5668c3fd9022ef4aac8287b82ab71124',
                'Content-Type':
                'application/x-www-form-urlencoded',
                'x-amz-date':
                '20160618T220405Z',
            }, mock_request.headers)
Example #32
0
def lambda_handler(event, context):
    es_host = os.environ['ELASTIC_SEARCH_HOSTNAME']
    num_indexes_to_keep = int(os.environ['NUM_INDEXES_TO_KEEP'])
    index_prefix = os.environ['INDEX_PREFIX']
    
    auth = AWSRequestsAuth(aws_access_key=os.environ['AWS_ACCESS_KEY_ID'],
                           aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
                           aws_token=os.environ['AWS_SESSION_TOKEN'],
                           aws_region=os.environ['AWS_REGION'],
                           aws_service='es',
                           aws_host=es_host)

    indexResponse = requests.get(f"https://{es_host}/*", auth=auth)
    if (indexResponse.status_code != 200):
        raise Exception(f"failed to retrieve indexes: {indexResponse.text}")
        
    indexData = indexResponse.json()
    indexNames = sorted([x for x in indexData.keys() if x.startswith(index_prefix)])
    indexesToDelete = indexNames[0 : max(0, len(indexNames) - num_indexes_to_keep)]

    print(f"number of selected indexes: {len(indexNames)}, number to delete: {len(indexesToDelete)}")
    
    for idx in indexesToDelete:
        deleteResponse = requests.delete(f"https://{es_host}/{idx}", auth=auth)
        if deleteResponse.status_code == 200:
            print(f"deleted {idx}")
        else:
            raise Exception(f"failed to delete {idx}: {deleteResponse.text}")
Example #33
0
    def __create_connection(self, config):
        kwargs = {
            'host': config['HOST'],
            'port': config.get('PORT', 9200),
            'use_ssl': config.get('USE_SSL', False),
            'verify_certs': True,
            'ca_certs': certifi.where()
        }

        if 'AWS_ACCESS_KEY' in config and \
           'AWS_SECRET_KEY' in config and \
           'AWS_REGION' in config:

            kwargs['connection_class'] = RequestsHttpConnection
            kwargs['http_auth'] = AWSRequestsAuth(
                aws_access_key=config['AWS_ACCESS_KEY'],
                aws_secret_access_key=config['AWS_SECRET_KEY'],
                aws_host=config['HOST'],
                aws_region=config['AWS_REGION'],
                aws_service='es')

        es = Elasticsearch(**kwargs)
        es._index = config['INDEX_NAME']
        es._settings = config.get('INDEX_SETTINGS', DEFAULT_INDEX_SETTINGS)
        return es
 def test_multiple_get_params(self):
     """
     Assert we generate the 'correct' cannonical query string
     for request that includes more than one query parameter
     """
     url = 'http://search-foo.us-east-1.es.amazonaws.com:80/index/type/_search?scroll=5m&search_type=scan'
     mock_request = mock.Mock()
     mock_request.url = url
     self.assertEqual('scroll=5m&search_type=scan', AWSRequestsAuth.get_canonical_querystring(mock_request))
 def test_post_request_with_get_param(self):
     """
     Assert we generate the 'correct' cannonical query string
     for a post request that includes GET-parameters
     """
     url = 'http://search-foo.us-east-1.es.amazonaws.com:80/index/type/1/_update?version=1'
     mock_request = mock.Mock()
     mock_request.url = url
     mock_request.method = "POST"
     self.assertEqual('version=1', AWSRequestsAuth.get_canonical_querystring(mock_request))