def __init__(self,
name,
region,
aws_access_key_id=None,
aws_secret_access_key=None,
debug=False,
kwargs=None):
self.settings = kwargs
self.pending = []
if kwargs.region not in [r.name for r in sqs.regions()]:
Log.error("Can not find region {{region}} in {{regions}}",
region=kwargs.region,
regions=[r.name for r in sqs.regions()])
conn = sqs.connect_to_region(
region_name=unwrap(kwargs.region),
aws_access_key_id=unwrap(kwargs.aws_access_key_id),
aws_secret_access_key=unwrap(kwargs.aws_secret_access_key),
)
self.queue = conn.get_queue(kwargs.name)
if self.queue == None:
Log.error(
"Can not find queue with name {{queue}} in region {{region}}",
queue=kwargs.name,
region=kwargs.region)
def test_credentialled_SQS_access(self):
"check that the credentials can access the SQS service"
creds = aws_credentials.get_credentials()
region_name = 'eu-west-1'
region = [r for r in regions() if r.name == region_name][0]
conn = SQSConnection(aws_access_key_id=creds[0], aws_secret_access_key=creds[1], region=region)
q = conn.create_queue("PLEASE_KEEP_FOR_TESTING", 30)
assert q
def get_all_regions(self):
"""
Fake - replacement for 'Connection.get_all_regions'
Returns list of AWS regions, using regular method 'boto.sqs.regions()'
"""
_ret_val = regions()
print '>>> Connection.get_all_regions()'
print '... return {0}'.format(_ret_val)
return _ret_val
def get_all_regions(self):
"""
Fake - replacement for 'Connection.get_all_regions'
Returns list of AWS regions, using regular method 'boto.sqs.regions()'
"""
_ret_val = regions()
print '>>> Connection.get_all_regions()'
print '... return {0}'.format(_ret_val)
return _ret_val
def get_sqs_region(region_name):
if region_name == '':
return None
for obj in sqs.regions():
if obj.name == region_name:
region_obj = obj
break
return region_obj
def slurp(self):
"""
:returns: item_list - list of SQS Policies.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
for region in regions():
app.logger.debug("Checking {}/{}/{}".format(SQS.index, account, region.name))
try:
sqs = connect(account, 'sqs', region=region)
all_queues = self.wrap_aws_rate_limited_call(
sqs.get_all_queues
)
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), 'sqs', account, region.name)
self.slurp_exception((self.index, account, region.name), exc, exception_map)
continue
app.logger.debug("Found {} {}".format(len(all_queues), SQS.i_am_plural))
for q in all_queues:
### Check if this Queue is on the Ignore List ###
ignore_item = False
for ignore_item_name in IGNORE_PREFIX[self.index]:
if q.name.lower().startswith(ignore_item_name.lower()):
ignore_item = True
break
if ignore_item:
continue
try:
policy = self.wrap_aws_rate_limited_call(
q.get_attributes,
attributes='Policy'
)
if 'Policy' in policy:
try:
json_str = policy['Policy']
policy = json.loads(json_str)
item = SQSItem(region=region.name, account=account, name=q.name,
config=policy)
item_list.append(item)
except:
self.slurp_exception((self.index, account, region, q.name), InvalidAWSJSON(json_str), exception_map)
except boto.exception.SQSError:
# A number of Queues are so ephemeral that they may be gone by the time
# the code reaches here. Just ignore them and move on.
pass
return item_list, exception_map
def __init__(self, name, acc_sec_pair=None, visibility_timeout=30):
self.name = name
if acc_sec_pair is None:
acc_sec_pair = aws_credentials.get_credentials()
self.region_name = 'eu-west-1'
self.region = [r for r in regions() if r.name == self.region_name][0]
self.conn = SQSConnection(aws_access_key_id=acc_sec_pair[0],
aws_secret_access_key=acc_sec_pair[1],
region=self.region)
self.q = self.conn.create_queue(name, visibility_timeout)
self.q.set_message_class(event.SQSEvent)
def __init__(self, name, acc_sec_pair=None, visibility_timeout=30):
self.name = name
if acc_sec_pair is None:
acc_sec_pair = aws_credentials.get_credentials()
self.region_name = 'eu-west-1'
self.region = [r for r in regions() if r.name == self.region_name][0]
self.conn = SQSConnection(aws_access_key_id=acc_sec_pair[0],
aws_secret_access_key=acc_sec_pair[1],
region=self.region)
self.q = self.conn.create_queue(name, visibility_timeout)
self.q.set_message_class(event.SQSEvent)
def sqs_connection(region_name=settings.AWS_SQS_REGION_NAME):
"""Returns an SQSConnection that is already authenticated.
us-west-2 is oregon, the region we use by default.
@raises IndexError
"""
region = filter(lambda x: x.name == region_name, sqs.regions())[0]
return sqs.connect_to_region(
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
region_name=region_name)
def setUp(self):
# Ensure the queue is clear before we start, or we'll lose more hair
creds = aws_credentials.get_credentials()
region_name = 'eu-west-1'
region = [r for r in regions() if r.name == region_name][0]
conn = SQSConnection(aws_access_key_id=creds[0],
aws_secret_access_key=creds[1],
region=region)
q = conn.create_queue(self.qname, 30)
cruft = q.get_messages(10)
while cruft:
for c in cruft:
print 'deleting old message %s', c.__dict__
q.delete_message(c)
cruft = q.get_messages(10)
def setUp(self):
# Ensure the queue is clear before we start, or we'll lose more hair
creds = aws_credentials.get_credentials()
region_name = 'eu-west-1'
region = [r for r in regions() if r.name == region_name][0]
conn = SQSConnection(aws_access_key_id=creds[0],
aws_secret_access_key=creds[1],
region=region)
q = conn.create_queue(self.qname, 30)
cruft = q.get_messages(10)
while cruft:
for c in cruft:
print 'deleting old message %s', c.__dict__
q.delete_message(c)
cruft = q.get_messages(10)
def __init__(
self,
name,
region,
aws_access_key_id=None,
aws_secret_access_key=None,
debug=False,
kwargs=None
):
self.settings = kwargs
self.pending = []
if kwargs.region not in [r.name for r in sqs.regions()]:
Log.error("Can not find region {{region}} in {{regions}}", region=kwargs.region, regions=[r.name for r in sqs.regions()])
conn = sqs.connect_to_region(
region_name=unwrap(kwargs.region),
aws_access_key_id=unwrap(kwargs.aws_access_key_id),
aws_secret_access_key=unwrap(kwargs.aws_secret_access_key),
)
self.queue = conn.get_queue(kwargs.name)
if self.queue == None:
Log.error("Can not find queue with name {{queue}} in region {{region}}", queue=kwargs.name, region=kwargs.region)
def sqs(self):
if self._sqs is None:
self._sqs = self._aws_connect_to(SQSConnection, _sqs.regions())
return self._sqs
def __init__(self, aws_access_key_id, aws_secret_access_key, region,
queue_name):
super(AwsSQSTools, self).__init__(aws_access_key_id,
aws_secret_access_key,
region=[i for i in regions() if i.name == region][0])
self.queue = self.create_queue(queue_name)
def _get_region(self, name):
return [r for r in regions() if r.name == name][0]
def sqs(self):
if self._sqs is None:
self._sqs = self._aws_connect_to(SQSConnection, _sqs.regions())
return self._sqs
def _get_region(self, name):
return [r for r in regions() if r.name == name][0]
import time
import threading
import S3
import Request
from boto.sqs.message import Message
from boto.sqs.connection import SQSConnection
import aws_common
from boto.sqs import regions
import xml.etree.ElementTree as ET
if __name__ == "__main__":
eu_region = None
for region in regions():
if region.name == 'eu-west-1':
eu_region = region
break
print eu_region
conn = SQSConnection(aws_access_key_id=aws_common.AWS_ACCESS_KEY,
aws_secret_access_key=aws_common.AWS_SECRET_KEY,
region=eu_region)
inQueue = conn.create_queue("12_LP1_SQS_D7001D_jimnys-8_frontend-in")
m = Message()
m.set_body("""
<RequestIDCellIDError>
<RequestType>CellStatNet</RequestType>
<TimeStart>201201011230</TimeStart>
<TimeStop>201301011250</TimeStop>
def slurp(self):
"""
:returns: item_list - list of SQS Policies.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
account_db = Account.query.filter(Account.name == account).first()
account_number = account_db.identifier
for region in regions():
app.logger.debug("Checking {}/{}/{}".format(
SQS.index, account, region.name))
try:
sqs = connect(account, 'sqs', region=region)
all_queues = self.wrap_aws_rate_limited_call(
sqs.get_all_queues)
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), 'sqs', account,
region.name)
self.slurp_exception(
(self.index, account, region.name),
exc,
exception_map,
source="{}-watcher".format(self.index))
continue
app.logger.debug("Found {} {}".format(len(all_queues),
SQS.i_am_plural))
for q in all_queues:
if self.check_ignore_list(q.name):
continue
try:
policy = self.wrap_aws_rate_limited_call(
q.get_attributes, attributes='Policy')
if 'Policy' in policy:
try:
arn = 'arn:aws:sqs:{region}:{account_number}:{name}'.format(
region=region.name,
account_number=account_number,
name=q.name)
json_str = policy['Policy']
policy = json.loads(json_str)
policy['arn'] = arn
item = SQSItem(region=region.name,
account=account,
name=q.name,
arn=arn,
config=policy)
item_list.append(item)
except:
self.slurp_exception(
(self.index, account, region, q.name),
InvalidAWSJSON(json_str),
exception_map,
source="{}-watcher".format(self.index))
except boto.exception.SQSError:
# A number of Queues are so ephemeral that they may be gone by the time
# the code reaches here. Just ignore them and move on.
pass
return item_list, exception_map
def sqs(self):
if self._sqs is None:
self._sqs = self._aws_connect_to(boto.connect_sqs, _sqs.regions())
return self._sqs
def slurp(self):
"""
:returns: item_list - list of SQS Policies.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
account_db = Account.query.filter(Account.name == account).first()
account_number = account_db.identifier
for region in regions():
app.logger.debug("Checking {}/{}/{}".format(SQS.index, account, region.name))
try:
sqs = connect(account, 'sqs', region=region)
all_queues = self.wrap_aws_rate_limited_call(
sqs.get_all_queues
)
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), 'sqs', account, region.name)
self.slurp_exception((self.index, account, region.name), exc, exception_map,
source="{}-watcher".format(self.index))
continue
app.logger.debug("Found {} {}".format(len(all_queues), SQS.i_am_plural))
for q in all_queues:
if self.check_ignore_list(q.name):
continue
try:
policy = self.wrap_aws_rate_limited_call(
q.get_attributes,
attributes='Policy'
)
if 'Policy' in policy:
try:
arn = 'arn:aws:sqs:{region}:{account_number}:{name}'.format(
region=region.name,
account_number=account_number,
name=q.name)
json_str = policy['Policy']
policy = json.loads(json_str)
policy['arn'] = arn
item = SQSItem(region=region.name, account=account, name=q.name, arn=arn,
config=policy)
item_list.append(item)
except:
self.slurp_exception((self.index, account, region, q.name), InvalidAWSJSON(json_str),
exception_map, source="{}-watcher".format(self.index))
except boto.exception.SQSError:
# A number of Queues are so ephemeral that they may be gone by the time
# the code reaches here. Just ignore them and move on.
pass
return item_list, exception_map
