def delete_all_queues(self, prefix): queues = self.get_queues() for url in queues: if prefix in url: print("Deleting queue '{}'".format(url)) retry.try_with_backoff(self.__context, self.__client.delete_queue, QueueUrl=url)
def get_query_execution(self, id): params = dict({}) params['QueryExecutionId'] = id response = retry.try_with_backoff(self.__context, self.__client.get_query_execution, **params) return response['QueryExecution']
def get_queues(self): response = retry.try_with_backoff(self.__context, self.__client.list_queues, QueueNamePrefix=self.__queue_prefix ) if 'QueueUrls' not in response: print "There are no queues found using queue prefix '{}'".format(self.__queue_prefix) return [] return response['QueueUrls']
def get_max_message_size(self): response = retry.try_with_backoff(self.__context, self.__client.get_queue_attributes, QueueUrl=self.__queue_url, AttributeNames=['All']) if response is None: return [] return response['Attributes']['MaximumMessageSize']
def send_generic_message(self, data): params = dict({}) params[c.SQS_PARAM_DELAY_SECONDS] = 0 params[c.SQS_PARAM_QUEUE_URL] = self.queue_url params["MessageBody"] = data response = retry.try_with_backoff(self.__context, self.__client.send_message, **params) return response
def send_message(self, data, sensitivity_type, compression_mode, payload_type): params = self.__create_message(sensitivity_type, data, compression_mode, payload_type) params[c.SQS_PARAM_QUEUE_URL] = self.queue_url response = retry.try_with_backoff(self.__context, self.__client.send_message, **params) return response
def get_partitions(self, db_name, table_name, token): params = dict({}) params['DatabaseName'] = db_name params['TableName'] = table_name if token: params['NextToken'] = token return retry.try_with_backoff({}, self.__client.get_partitions, **params)
def delete_partitions(self, db_name, table_name, values): if len(values) == 0: return None params = dict({}) params['DatabaseName'] = db_name params['TableName'] = table_name params['PartitionsToDelete'] = values print params return retry.try_with_backoff({}, self.__client.batch_delete_partition, **params)
def send_message_batch(self, sensitivity_type, data, compression_mode, payload_type): params = dict({}) params[c.SQS_PARAM_QUEUE_URL] = self.__queue_url params["Entries"] = [] for message in data: batch_item = self.__create_message(sensitivity_type, message, compression_mode, payload_type) batch_item["Id"] = uuid.uuid1().hex params["Entries"].append(batch_item) response=retry.try_with_backoff(self.__context, self.__client.send_message_batch, **params) return response
def get_queue_attributes(self, queue_url=None): if queue_url is None: queue_url = self.__queue_url return retry.try_with_backoff( self.__context, self.__client.get_queue_attributes, QueueUrl=queue_url, AttributeNames=[ 'ApproximateNumberOfMessagesNotVisible', 'ApproximateNumberOfMessages' ])
def read_queue(self): timeout = self.__context[c.KEY_MAX_LAMBDA_TIME] + 30 response = retry.try_with_backoff(self.__context, self.__client.receive_message, \ QueueUrl=self.__queue_url, \ AttributeNames=['ApproximateReceiveCount'], \ MessageAttributeNames=['All'], \ MaxNumberOfMessages=10, \ VisibilityTimeout=timeout, \ WaitTimeSeconds=5 #enables log polling http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-long-polling.html ) if response is None or 'Messages' not in response: return [] return response['Messages']
def query(self, sql, result_as_list=True, sync=True): if not self.is_valid_query(sql): return None print("Executing query\n\t", sql) params = dict({}) params['QueryString'] = sql params['QueryExecutionContext'] = {'Database': self.__db_name} params['ResultConfiguration'] = { 'OutputLocation': "s3://{}/{}".format(self.__bucket, self.query_results_path), 'EncryptionConfiguration': { 'EncryptionOption': 'SSE_S3' } } response = retry.try_with_backoff(self.__context, self.__client.start_query_execution, **params) id = response['QueryExecutionId'] if sync: #TODO: implement a boto3 waiter while True: query = self.get_query_execution(id) print("Query '{}...' is".format(sql[:30]), query['Status']['State']) if query['Status']['State'] == 'RUNNING' or query['Status'][ 'State'] == 'QUEUED': time.sleep(3) elif query['Status']['State'] == 'FAILED': print("The query '{}' FAILED with ERROR: {}".format( query, query['Status']["StateChangeReason"])) if 'HIVE_CANNOT_OPEN_SPLIT' in query['Status'][ "StateChangeReason"]: #The amoeba generator could be running which would cause files to be removed return [] else: return None else: return self.get_output( query['ResultConfiguration']['OutputLocation'], result_as_list) else: return id
def create_crawler(self, name, role, dbname, table_prefix="auto_", srcs=None, schedule=None): params = dict({}) params['Name'] = "{}".format(self.__name(name)).lower() params['Role'] = role params['DatabaseName'] = dbname #schedule every hour if schedule: params['Schedule'] = schedule #"Cron(0 0/1 * * ? *)" params['TablePrefix'] = table_prefix params['Targets'] = {'S3Targets': srcs} params[ 'Configuration'] = '{ "Version": 1.0, "CrawlerOutput": { "Partitions": { "AddOrUpdateBehavior": "InheritFromTable" } }}' response = retry.try_with_backoff({}, self.__client.create_crawler, **params) return response
def get_key(self, key): params = dict({}) params["KeyConditionExpression"] = Key('key').eq(key) response = retry.try_with_backoff(self.__context, self.context_table.query, **params) return json.loads(json.dumps(response, cls=util.DynamoDbDecoder))
def get(self, table, params={}): response = retry.try_with_backoff(self.__context, table.scan, **params) return json.loads(json.dumps(response, cls=util.DynamoDbDecoder))
def put_metric_data(self, namespace, metric_data): params = {'Namespace': namespace, 'MetricData': metric_data} return retry.try_with_backoff(self.__context, self.__client.put_metric_data, **params)