def __init__(self, name, region, aws_access_key_id=None, aws_secret_access_key=None, debug=False, kwargs=None): self.settings = kwargs self.pending = [] if kwargs.region not in [r.name for r in sqs.regions()]: Log.error("Can not find region {{region}} in {{regions}}", region=kwargs.region, regions=[r.name for r in sqs.regions()]) conn = sqs.connect_to_region( region_name=unwrap(kwargs.region), aws_access_key_id=unwrap(kwargs.aws_access_key_id), aws_secret_access_key=unwrap(kwargs.aws_secret_access_key), ) self.queue = conn.get_queue(kwargs.name) if self.queue == None: Log.error( "Can not find queue with name {{queue}} in region {{region}}", queue=kwargs.name, region=kwargs.region)
def test_credentialled_SQS_access(self): "check that the credentials can access the SQS service" creds = aws_credentials.get_credentials() region_name = 'eu-west-1' region = [r for r in regions() if r.name == region_name][0] conn = SQSConnection(aws_access_key_id=creds[0], aws_secret_access_key=creds[1], region=region) q = conn.create_queue("PLEASE_KEEP_FOR_TESTING", 30) assert q
def get_all_regions(self): """ Fake - replacement for 'Connection.get_all_regions' Returns list of AWS regions, using regular method 'boto.sqs.regions()' """ _ret_val = regions() print '>>> Connection.get_all_regions()' print '... return {0}'.format(_ret_val) return _ret_val
def get_sqs_region(region_name): if region_name == '': return None for obj in sqs.regions(): if obj.name == region_name: region_obj = obj break return region_obj
def slurp(self): """ :returns: item_list - list of SQS Policies. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: for region in regions(): app.logger.debug("Checking {}/{}/{}".format(SQS.index, account, region.name)) try: sqs = connect(account, 'sqs', region=region) all_queues = self.wrap_aws_rate_limited_call( sqs.get_all_queues ) except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), 'sqs', account, region.name) self.slurp_exception((self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}".format(len(all_queues), SQS.i_am_plural)) for q in all_queues: ### Check if this Queue is on the Ignore List ### ignore_item = False for ignore_item_name in IGNORE_PREFIX[self.index]: if q.name.lower().startswith(ignore_item_name.lower()): ignore_item = True break if ignore_item: continue try: policy = self.wrap_aws_rate_limited_call( q.get_attributes, attributes='Policy' ) if 'Policy' in policy: try: json_str = policy['Policy'] policy = json.loads(json_str) item = SQSItem(region=region.name, account=account, name=q.name, config=policy) item_list.append(item) except: self.slurp_exception((self.index, account, region, q.name), InvalidAWSJSON(json_str), exception_map) except boto.exception.SQSError: # A number of Queues are so ephemeral that they may be gone by the time # the code reaches here. Just ignore them and move on. pass return item_list, exception_map
def __init__(self, name, acc_sec_pair=None, visibility_timeout=30): self.name = name if acc_sec_pair is None: acc_sec_pair = aws_credentials.get_credentials() self.region_name = 'eu-west-1' self.region = [r for r in regions() if r.name == self.region_name][0] self.conn = SQSConnection(aws_access_key_id=acc_sec_pair[0], aws_secret_access_key=acc_sec_pair[1], region=self.region) self.q = self.conn.create_queue(name, visibility_timeout) self.q.set_message_class(event.SQSEvent)
def sqs_connection(region_name=settings.AWS_SQS_REGION_NAME): """Returns an SQSConnection that is already authenticated. us-west-2 is oregon, the region we use by default. @raises IndexError """ region = filter(lambda x: x.name == region_name, sqs.regions())[0] return sqs.connect_to_region( aws_access_key_id=settings.AWS_ACCESS_KEY_ID, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, region_name=region_name)
def setUp(self): # Ensure the queue is clear before we start, or we'll lose more hair creds = aws_credentials.get_credentials() region_name = 'eu-west-1' region = [r for r in regions() if r.name == region_name][0] conn = SQSConnection(aws_access_key_id=creds[0], aws_secret_access_key=creds[1], region=region) q = conn.create_queue(self.qname, 30) cruft = q.get_messages(10) while cruft: for c in cruft: print 'deleting old message %s', c.__dict__ q.delete_message(c) cruft = q.get_messages(10)
def __init__( self, name, region, aws_access_key_id=None, aws_secret_access_key=None, debug=False, kwargs=None ): self.settings = kwargs self.pending = [] if kwargs.region not in [r.name for r in sqs.regions()]: Log.error("Can not find region {{region}} in {{regions}}", region=kwargs.region, regions=[r.name for r in sqs.regions()]) conn = sqs.connect_to_region( region_name=unwrap(kwargs.region), aws_access_key_id=unwrap(kwargs.aws_access_key_id), aws_secret_access_key=unwrap(kwargs.aws_secret_access_key), ) self.queue = conn.get_queue(kwargs.name) if self.queue == None: Log.error("Can not find queue with name {{queue}} in region {{region}}", queue=kwargs.name, region=kwargs.region)
def sqs(self): if self._sqs is None: self._sqs = self._aws_connect_to(SQSConnection, _sqs.regions()) return self._sqs
def __init__(self, aws_access_key_id, aws_secret_access_key, region, queue_name): super(AwsSQSTools, self).__init__(aws_access_key_id, aws_secret_access_key, region=[i for i in regions() if i.name == region][0]) self.queue = self.create_queue(queue_name)
def _get_region(self, name): return [r for r in regions() if r.name == name][0]
import time import threading import S3 import Request from boto.sqs.message import Message from boto.sqs.connection import SQSConnection import aws_common from boto.sqs import regions import xml.etree.ElementTree as ET if __name__ == "__main__": eu_region = None for region in regions(): if region.name == 'eu-west-1': eu_region = region break print eu_region conn = SQSConnection(aws_access_key_id=aws_common.AWS_ACCESS_KEY, aws_secret_access_key=aws_common.AWS_SECRET_KEY, region=eu_region) inQueue = conn.create_queue("12_LP1_SQS_D7001D_jimnys-8_frontend-in") m = Message() m.set_body(""" <RequestIDCellIDError> <RequestType>CellStatNet</RequestType> <TimeStart>201201011230</TimeStart> <TimeStop>201301011250</TimeStop>
def slurp(self): """ :returns: item_list - list of SQS Policies. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: account_db = Account.query.filter(Account.name == account).first() account_number = account_db.identifier for region in regions(): app.logger.debug("Checking {}/{}/{}".format( SQS.index, account, region.name)) try: sqs = connect(account, 'sqs', region=region) all_queues = self.wrap_aws_rate_limited_call( sqs.get_all_queues) except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), 'sqs', account, region.name) self.slurp_exception( (self.index, account, region.name), exc, exception_map, source="{}-watcher".format(self.index)) continue app.logger.debug("Found {} {}".format(len(all_queues), SQS.i_am_plural)) for q in all_queues: if self.check_ignore_list(q.name): continue try: policy = self.wrap_aws_rate_limited_call( q.get_attributes, attributes='Policy') if 'Policy' in policy: try: arn = 'arn:aws:sqs:{region}:{account_number}:{name}'.format( region=region.name, account_number=account_number, name=q.name) json_str = policy['Policy'] policy = json.loads(json_str) policy['arn'] = arn item = SQSItem(region=region.name, account=account, name=q.name, arn=arn, config=policy) item_list.append(item) except: self.slurp_exception( (self.index, account, region, q.name), InvalidAWSJSON(json_str), exception_map, source="{}-watcher".format(self.index)) except boto.exception.SQSError: # A number of Queues are so ephemeral that they may be gone by the time # the code reaches here. Just ignore them and move on. pass return item_list, exception_map
def sqs(self): if self._sqs is None: self._sqs = self._aws_connect_to(boto.connect_sqs, _sqs.regions()) return self._sqs
def slurp(self): """ :returns: item_list - list of SQS Policies. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: account_db = Account.query.filter(Account.name == account).first() account_number = account_db.identifier for region in regions(): app.logger.debug("Checking {}/{}/{}".format(SQS.index, account, region.name)) try: sqs = connect(account, 'sqs', region=region) all_queues = self.wrap_aws_rate_limited_call( sqs.get_all_queues ) except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), 'sqs', account, region.name) self.slurp_exception((self.index, account, region.name), exc, exception_map, source="{}-watcher".format(self.index)) continue app.logger.debug("Found {} {}".format(len(all_queues), SQS.i_am_plural)) for q in all_queues: if self.check_ignore_list(q.name): continue try: policy = self.wrap_aws_rate_limited_call( q.get_attributes, attributes='Policy' ) if 'Policy' in policy: try: arn = 'arn:aws:sqs:{region}:{account_number}:{name}'.format( region=region.name, account_number=account_number, name=q.name) json_str = policy['Policy'] policy = json.loads(json_str) policy['arn'] = arn item = SQSItem(region=region.name, account=account, name=q.name, arn=arn, config=policy) item_list.append(item) except: self.slurp_exception((self.index, account, region, q.name), InvalidAWSJSON(json_str), exception_map, source="{}-watcher".format(self.index)) except boto.exception.SQSError: # A number of Queues are so ephemeral that they may be gone by the time # the code reaches here. Just ignore them and move on. pass return item_list, exception_map