def __init__(self, domain=None, endpoint=None):
self.domain = domain
self.endpoint = endpoint
self.session = requests.Session()
# Endpoint needs to be set before initializing CloudSearchDomainConnection
if not endpoint:
self.endpoint = domain.search_service_endpoint
# Copy proxy settings from connection and check if request should be signed
self.sign_request = False
if self.domain and self.domain.layer1:
if self.domain.layer1.use_proxy:
self.session.proxies['http'] = self.domain.layer1.get_proxy_url_with_auth()
self.sign_request = getattr(self.domain.layer1, 'sign_request', False)
if self.sign_request:
layer1 = self.domain.layer1
self.domain_connection = CloudSearchDomainConnection(
host=self.endpoint,
aws_access_key_id=layer1.aws_access_key_id,
aws_secret_access_key=layer1.aws_secret_access_key,
region=layer1.region,
provider=layer1.provider
)
def __init__(self, domain=None, endpoint=None):
self.domain = domain
self.endpoint = endpoint
if not self.endpoint:
self.endpoint = domain.doc_service_endpoint
self.documents_batch = []
self._sdf = None
# Copy proxy settings from connection and check if request should be signed
self.proxy = {}
self.sign_request = False
if self.domain and self.domain.layer1:
if self.domain.layer1.use_proxy:
self.proxy = {
'http': self.domain.layer1.get_proxy_url_with_auth()
}
self.sign_request = getattr(self.domain.layer1, 'sign_request',
False)
if self.sign_request:
# Create a domain connection to send signed requests
layer1 = self.domain.layer1
self.domain_connection = CloudSearchDomainConnection(
host=self.endpoint,
aws_access_key_id=layer1.aws_access_key_id,
aws_secret_access_key=layer1.aws_secret_access_key,
region=layer1.region,
provider=layer1.provider)
def connect_cloudsearchdomain(aws_access_key_id=None,
aws_secret_access_key=None,
**kwargs):
"""
:type aws_access_key_id: string
:param aws_access_key_id: Your AWS Access Key ID
:type aws_secret_access_key: string
:param aws_secret_access_key: Your AWS Secret Access Key
:rtype: :class:`boto.cloudsearchdomain.layer1.CloudSearchDomainConnection`
:return: A connection to Amazon's CloudSearch Domain service
"""
from boto.cloudsearchdomain.layer1 import CloudSearchDomainConnection
return CloudSearchDomainConnection(aws_access_key_id,
aws_secret_access_key, **kwargs)
def delete_doc_in_cloudsearch(doc_ids,
cloudsearch_domain_name,
validate=True,
region='ap-southeast-2'):
"""
:param doc_ids: the doc id array
:param cloudsearch_domain_name: the domain name
!!! This method is in-efficient for validating.
"""
conn = boto.cloudsearch2.connect_to_region(region)
domain = conn.describe_domains([cloudsearch_domain_name]) \
['DescribeDomainsResponse']['DescribeDomainsResult']['DomainStatusList']
doc_service = CloudSearchDomainConnection(
host=domain[0]['DocService']['Endpoint'], region=region)
if len(doc_ids) > 0:
batch = []
for id in doc_ids:
batch.append({"id": str(id), "type": "delete"})
doc_service.upload_documents(json.dumps(batch), 'application/json')
if validate:
# we need some time for the cloudsearch to apply the deletion.
MAX_RETRIES = 100
SLEEP_GAP = 10
for id in doc_ids:
retry = 0
while retry < MAX_RETRIES:
result = doc_service.search(query="(term field=_id '%s')" % id,
query_parser='structured')
if (result['hits']['found'] == 0):
break
retry += 1
sleep(SLEEP_GAP)
if retry >= MAX_RETRIES:
raise ValueError(
"Intended to delete cloudsearch documents with id %s, but it still exists after deletion. "
"Culprit document: %s" % (doc_ids, id))
def test_no_host_provided(self):
# A host must be provided or a error is thrown.
with self.assertRaises(ValueError):
CloudSearchDomainConnection(
aws_access_key_id='aws_access_key_id',
aws_secret_access_key='aws_secret_access_key')
