def authenticate(self):
# This value controls how long we sleep
# between reauthenticating and getting a new set of creds
# eventually this gets set by aws response
self.flush_wait_time = 1800
if options.cloudtrail_arn not in [
"<cloudtrail_arn>", "cloudtrail_arn"
]:
client = boto3.client("sts",
aws_access_key_id=options.accesskey,
aws_secret_access_key=options.secretkey)
response = client.assume_role(
RoleArn=options.cloudtrail_arn,
RoleSessionName="MozDef-CloudTrail-Reader")
role_creds = {
"aws_access_key_id": response["Credentials"]["AccessKeyId"],
"aws_secret_access_key":
response["Credentials"]["SecretAccessKey"],
"aws_session_token": response["Credentials"]["SessionToken"],
}
current_time = toUTC(datetime.now())
# Let's remove 3 seconds from the flush wait time just in case
self.flush_wait_time = (response["Credentials"]["Expiration"] -
current_time).seconds - 3
else:
role_creds = {}
role_creds["region_name"] = options.region
self.s3_client = boto3.client("s3",
**get_aws_credentials(**role_creds))
def authenticate(self):
# This value controls how long we sleep
# between reauthenticating and getting a new set of creds
# eventually this gets set by aws response
self.flush_wait_time = 1800
if options.cloudtrail_arn not in ['<cloudtrail_arn>', 'cloudtrail_arn']:
client = boto3.client(
'sts',
aws_access_key_id=options.accesskey,
aws_secret_access_key=options.secretkey
)
response = client.assume_role(
RoleArn=options.cloudtrail_arn,
RoleSessionName='MozDef-CloudTrail-Reader',
)
role_creds = {
'aws_access_key_id': response['Credentials']['AccessKeyId'],
'aws_secret_access_key': response['Credentials']['SecretAccessKey'],
'aws_session_token': response['Credentials']['SessionToken']
}
current_time = toUTC(datetime.now())
# Let's remove 3 seconds from the flush wait time just in case
self.flush_wait_time = (response['Credentials']['Expiration'] - current_time).seconds - 3
else:
role_creds = {}
role_creds['region_name'] = options.region
self.s3_client = boto3.client(
's3',
**get_aws_credentials(**role_creds)
)
def authenticate(self):
if options.cloudtrail_arn not in [
'<cloudtrail_arn>', 'cloudtrail_arn'
]:
role_manager = RoleManager(**get_aws_credentials(
options.region, options.accesskey, options.secretkey))
role_manager.assume_role(options.cloudtrail_arn)
role_creds = role_manager.get_credentials(options.cloudtrail_arn)
else:
role_creds = {}
self.s3_connection = boto.connect_s3(**role_creds)
def authenticate(self):
if options.cloudtrail_arn not in ['<cloudtrail_arn>', 'cloudtrail_arn']:
role_manager = RoleManager(**get_aws_credentials(
options.region,
options.accesskey,
options.secretkey))
role_manager.assume_role(options.cloudtrail_arn)
role_creds = role_manager.get_credentials(options.cloudtrail_arn)
else:
role_creds = {}
self.s3_connection = boto.connect_s3(**role_creds)
def run(self):
self.taskQueue.set_message_class(RawMessage)
while True:
try:
records = self.taskQueue.get_messages(options.prefetch)
for msg in records:
body_message = msg.get_body()
event = json.loads(body_message)
if not event['Message']:
logger.error(
'Invalid message format for cloudtrail SQS messages'
)
logger.error('Malformed Message: %r' % body_message)
continue
if event['Message'] == 'CloudTrail validation message.':
# We don't care about these messages
continue
message_json = json.loads(event['Message'])
if 's3ObjectKey' not in message_json:
logger.error(
'Invalid message format, expecting an s3ObjectKey in Message'
)
logger.error('Malformed Message: %r' % body_message)
continue
s3_log_files = message_json['s3ObjectKey']
for log_file in s3_log_files:
logger.debug('Downloading and parsing ' + log_file)
bucket = self.s3_connection.get_bucket(
message_json['s3Bucket'])
log_file_lookup = bucket.lookup(log_file)
events = self.process_file(log_file_lookup)
for event in events:
self.on_message(event)
self.taskQueue.delete_message(msg)
except (SSLEOFError, SSLError, socket.error):
logger.info('Received network related error...reconnecting')
time.sleep(5)
self.connection, self.taskQueue = connect_sqs(
task_exchange=options.taskexchange,
**get_aws_credentials(options.region, options.accesskey,
options.secretkey))
self.taskQueue.set_message_class(RawMessage)
def __init__(self, region_name='us-east-1', aws_access_key_id=None, aws_secret_access_key=None):
self.aws_access_key_id = aws_access_key_id
self.aws_secret_access_key = aws_secret_access_key
self.credentials = {}
self.session_credentials = None
self.session_conn_sts = None
try:
self.local_conn_sts = boto.sts.connect_to_region(
**get_aws_credentials(
region_name,
self.aws_access_key_id,
self.aws_secret_access_key))
except Exception, e:
logger.error("Unable to connect to STS due to exception %s" % e.message)
raise
def main():
if hasUWSGI:
logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id()))
else:
logger.info('started without uwsgi')
if options.mqprotocol not in ('sqs'):
logger.error('Can only process SQS queues, terminating')
sys.exit(1)
sqs_conn, eventTaskQueue = connect_sqs(
task_exchange=options.taskexchange,
**get_aws_credentials(options.region, options.accesskey,
options.secretkey))
# consume our queue
taskConsumer(sqs_conn, eventTaskQueue, es, options).run()
def __init__(self,
region_name='us-east-1',
aws_access_key_id=None,
aws_secret_access_key=None):
self.aws_access_key_id = aws_access_key_id
self.aws_secret_access_key = aws_secret_access_key
self.credentials = {}
self.session_credentials = None
self.session_conn_sts = None
try:
self.local_conn_sts = boto.sts.connect_to_region(
**get_aws_credentials(region_name, self.aws_access_key_id,
self.aws_secret_access_key))
except Exception, e:
logger.error("Unable to connect to STS due to exception %s" %
e.message)
raise
def run(self):
self.taskQueue.set_message_class(RawMessage)
while True:
try:
records = self.taskQueue.get_messages(options.prefetch)
for msg in records:
body_message = msg.get_body()
event = json.loads(body_message)
if not event['Message']:
logger.error('Invalid message format for cloudtrail SQS messages')
logger.error('Malformed Message: %r' % body_message)
continue
if event['Message'] == 'CloudTrail validation message.':
# We don't care about these messages
continue
message_json = json.loads(event['Message'])
if 's3ObjectKey' not in message_json:
logger.error('Invalid message format, expecting an s3ObjectKey in Message')
logger.error('Malformed Message: %r' % body_message)
continue
s3_log_files = message_json['s3ObjectKey']
for log_file in s3_log_files:
logger.debug('Downloading and parsing ' + log_file)
bucket = self.s3_connection.get_bucket(message_json['s3Bucket'])
log_file_lookup = bucket.lookup(log_file)
events = self.process_file(log_file_lookup)
for event in events:
self.on_message(event)
self.taskQueue.delete_message(msg)
except (SSLEOFError, SSLError, socket.error):
logger.info('Received network related error...reconnecting')
time.sleep(5)
self.connection, self.taskQueue = connect_sqs(
task_exchange=options.taskexchange,
**get_aws_credentials(
options.region,
options.accesskey,
options.secretkey))
self.taskQueue.set_message_class(RawMessage)
def main():
if hasUWSGI:
logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id()))
else:
logger.info('started without uwsgi')
if options.mqprotocol not in ('sqs'):
logger.error('Can only process SQS queues, terminating')
sys.exit(1)
sqs_conn, eventTaskQueue = connect_sqs(
task_exchange=options.taskexchange,
**get_aws_credentials(
options.region,
options.accesskey,
options.secretkey))
# consume our queue
taskConsumer(sqs_conn, eventTaskQueue, es, options).run()
class RoleManager:
def __init__(self,
region_name='us-east-1',
aws_access_key_id=None,
aws_secret_access_key=None):
self.aws_access_key_id = aws_access_key_id
self.aws_secret_access_key = aws_secret_access_key
self.credentials = {}
self.session_credentials = None
self.session_conn_sts = None
try:
self.local_conn_sts = boto.sts.connect_to_region(
**get_aws_credentials(region_name, self.aws_access_key_id,
self.aws_secret_access_key))
except Exception, e:
logger.error("Unable to connect to STS due to exception %s" %
e.message)
raise
if self.aws_access_key_id is not None or self.aws_secret_access_key is not None:
# We're using API credentials not an IAM Role
try:
if self.session_credentials is None or self.session_credentials.is_expired(
):
self.session_credentials = self.local_conn_sts.get_session_token(
)
except Exception, e:
logger.error(
"Unable to get session token due to exception %s" %
e.message)
raise
try:
creds = get_aws_credentials(
region_name, self.session_credentials.access_key,
self.session_credentials.secret_key,
self.session_credentials.session_token
) if self.session_credentials else {}
self.session_conn_sts = boto.sts.connect_to_region(**creds)
except Exception, e:
logger.error(
"Unable to connect to STS with session token due to exception %s"
% e.message)
raise