def delete_all_queues(self, prefix):
queues = self.get_queues()
for url in queues:
if prefix in url:
print("Deleting queue '{}'".format(url))
retry.try_with_backoff(self.__context,
self.__client.delete_queue,
QueueUrl=url)
def get_query_execution(self, id):
params = dict({})
params['QueryExecutionId'] = id
response = retry.try_with_backoff(self.__context,
self.__client.get_query_execution,
**params)
return response['QueryExecution']
def get_queues(self):
response = retry.try_with_backoff(self.__context, self.__client.list_queues,
QueueNamePrefix=self.__queue_prefix
)
if 'QueueUrls' not in response:
print "There are no queues found using queue prefix '{}'".format(self.__queue_prefix)
return []
return response['QueueUrls']
def get_max_message_size(self):
response = retry.try_with_backoff(self.__context,
self.__client.get_queue_attributes,
QueueUrl=self.__queue_url,
AttributeNames=['All'])
if response is None:
return []
return response['Attributes']['MaximumMessageSize']
def send_generic_message(self, data):
params = dict({})
params[c.SQS_PARAM_DELAY_SECONDS] = 0
params[c.SQS_PARAM_QUEUE_URL] = self.queue_url
params["MessageBody"] = data
response = retry.try_with_backoff(self.__context,
self.__client.send_message, **params)
return response
def send_message(self, data, sensitivity_type, compression_mode,
payload_type):
params = self.__create_message(sensitivity_type, data,
compression_mode, payload_type)
params[c.SQS_PARAM_QUEUE_URL] = self.queue_url
response = retry.try_with_backoff(self.__context,
self.__client.send_message, **params)
return response
def get_partitions(self, db_name, table_name, token):
params = dict({})
params['DatabaseName'] = db_name
params['TableName'] = table_name
if token:
params['NextToken'] = token
return retry.try_with_backoff({}, self.__client.get_partitions,
**params)
def delete_partitions(self, db_name, table_name, values):
if len(values) == 0:
return None
params = dict({})
params['DatabaseName'] = db_name
params['TableName'] = table_name
params['PartitionsToDelete'] = values
print params
return retry.try_with_backoff({}, self.__client.batch_delete_partition,
**params)
def send_message_batch(self, sensitivity_type, data, compression_mode, payload_type):
params = dict({})
params[c.SQS_PARAM_QUEUE_URL] = self.__queue_url
params["Entries"] = []
for message in data:
batch_item = self.__create_message(sensitivity_type, message, compression_mode, payload_type)
batch_item["Id"] = uuid.uuid1().hex
params["Entries"].append(batch_item)
response=retry.try_with_backoff(self.__context, self.__client.send_message_batch, **params)
return response
def get_queue_attributes(self, queue_url=None):
if queue_url is None:
queue_url = self.__queue_url
return retry.try_with_backoff(
self.__context,
self.__client.get_queue_attributes,
QueueUrl=queue_url,
AttributeNames=[
'ApproximateNumberOfMessagesNotVisible',
'ApproximateNumberOfMessages'
])
def read_queue(self):
timeout = self.__context[c.KEY_MAX_LAMBDA_TIME] + 30
response = retry.try_with_backoff(self.__context, self.__client.receive_message, \
QueueUrl=self.__queue_url, \
AttributeNames=['ApproximateReceiveCount'], \
MessageAttributeNames=['All'], \
MaxNumberOfMessages=10, \
VisibilityTimeout=timeout, \
WaitTimeSeconds=5 #enables log polling http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-long-polling.html
)
if response is None or 'Messages' not in response:
return []
return response['Messages']
def query(self, sql, result_as_list=True, sync=True):
if not self.is_valid_query(sql):
return None
print("Executing query\n\t", sql)
params = dict({})
params['QueryString'] = sql
params['QueryExecutionContext'] = {'Database': self.__db_name}
params['ResultConfiguration'] = {
'OutputLocation':
"s3://{}/{}".format(self.__bucket, self.query_results_path),
'EncryptionConfiguration': {
'EncryptionOption': 'SSE_S3'
}
}
response = retry.try_with_backoff(self.__context,
self.__client.start_query_execution,
**params)
id = response['QueryExecutionId']
if sync:
#TODO: implement a boto3 waiter
while True:
query = self.get_query_execution(id)
print("Query '{}...' is".format(sql[:30]),
query['Status']['State'])
if query['Status']['State'] == 'RUNNING' or query['Status'][
'State'] == 'QUEUED':
time.sleep(3)
elif query['Status']['State'] == 'FAILED':
print("The query '{}' FAILED with ERROR: {}".format(
query, query['Status']["StateChangeReason"]))
if 'HIVE_CANNOT_OPEN_SPLIT' in query['Status'][
"StateChangeReason"]:
#The amoeba generator could be running which would cause files to be removed
return []
else:
return None
else:
return self.get_output(
query['ResultConfiguration']['OutputLocation'],
result_as_list)
else:
return id
def create_crawler(self,
name,
role,
dbname,
table_prefix="auto_",
srcs=None,
schedule=None):
params = dict({})
params['Name'] = "{}".format(self.__name(name)).lower()
params['Role'] = role
params['DatabaseName'] = dbname
#schedule every hour
if schedule:
params['Schedule'] = schedule #"Cron(0 0/1 * * ? *)"
params['TablePrefix'] = table_prefix
params['Targets'] = {'S3Targets': srcs}
params[
'Configuration'] = '{ "Version": 1.0, "CrawlerOutput": { "Partitions": { "AddOrUpdateBehavior": "InheritFromTable" } }}'
response = retry.try_with_backoff({}, self.__client.create_crawler,
**params)
return response
def get_key(self, key):
params = dict({})
params["KeyConditionExpression"] = Key('key').eq(key)
response = retry.try_with_backoff(self.__context,
self.context_table.query, **params)
return json.loads(json.dumps(response, cls=util.DynamoDbDecoder))
def get(self, table, params={}):
response = retry.try_with_backoff(self.__context, table.scan, **params)
return json.loads(json.dumps(response, cls=util.DynamoDbDecoder))
def put_metric_data(self, namespace, metric_data):
params = {'Namespace': namespace, 'MetricData': metric_data}
return retry.try_with_backoff(self.__context,
self.__client.put_metric_data, **params)
