def run(self):
while True:
try:
records = self.sqs_queue.receive_messages(
MaxNumberOfMessages=options.prefetch)
for msg in records:
msg_body = msg.body
try:
# get_body() should be json
message_json = json.loads(msg_body)
self.on_message(message_json)
# delete message from queue
msg.delete()
except ValueError:
logger.error(
'Invalid message, not JSON <dropping message and continuing>: %r'
% msg_body)
msg.delete()
continue
time.sleep(options.sleep_time)
except (SSLEOFError, SSLError, socket.error):
logger.info('Received network related error...reconnecting')
time.sleep(5)
self.sqs_queue = connect_sqs(options.region, options.accesskey,
options.secretkey,
options.taskexchange)
def run(self):
self.taskQueue.set_message_class(RawMessage)
while True:
try:
records = self.taskQueue.get_messages(self.options.prefetch)
for msg in records:
msg_body = msg.get_body()
try:
# get_body() should be json
message_json = json.loads(msg_body)
self.on_message(message_json)
# delete message from queue
self.taskQueue.delete_message(msg)
except ValueError:
logger.error('Invalid message, not JSON <dropping message and continuing>: %r' % msg_body)
self.taskQueue.delete_message(msg)
continue
time.sleep(.1)
except (SSLEOFError, SSLError, socket.error):
logger.info('Received network related error...reconnecting')
time.sleep(5)
self.connection, self.taskQueue = connect_sqs(
options.region,
options.accesskey,
options.secretkey,
options.taskexchange
)
self.taskQueue.set_message_class(RawMessage)
def run(self):
while True:
try:
records = self.sqs_queue.receive_messages(
MaxNumberOfMessages=options.prefetch)
for msg in records:
# msg.id is the id,
# get_body() should be json
# pre process the message a bit
tmp = msg.body
try:
msgbody = json.loads(tmp)
except ValueError:
# If Boto wrote to the queue, it might be base64 encoded, so let's decode that
try:
tmp = base64.b64decode(tmp)
msgbody = json.loads(tmp)
except Exception as e:
logger.error(
"Invalid message, not JSON <dropping message and continuing>: %r"
% msg.get_body())
msg.delete()
continue
# If this is still not a dict,
# let's just drop the message and move on
if type(msgbody) is not dict:
logger.debug(
"Message is not a dictionary, dropping message.")
msg.delete()
continue
event = dict()
event = msgbody
if "tags" in event:
event["tags"].extend([options.taskexchange])
else:
event["tags"] = [options.taskexchange]
# process message
self.on_message(event, msg)
# delete message from queue
msg.delete()
time.sleep(options.sleep_time)
except ValueError as e:
logger.exception("Exception while handling message: %r" % e)
msg.delete()
except (SSLEOFError, SSLError, socket.error):
logger.info("Received network related error...reconnecting")
time.sleep(5)
self.sqs_queue = connect_sqs(options.region, options.accesskey,
options.secretkey,
options.taskexchange)
def run(self):
self.taskQueue.set_message_class(RawMessage)
while True:
try:
records = self.taskQueue.get_messages(options.prefetch)
for msg in records:
body_message = msg.get_body()
event = json.loads(body_message)
if not event['Message']:
logger.error(
'Invalid message format for cloudtrail SQS messages'
)
logger.error('Malformed Message: %r' % body_message)
continue
if event['Message'] == 'CloudTrail validation message.':
# We don't care about these messages
continue
message_json = json.loads(event['Message'])
if 's3ObjectKey' not in message_json.keys():
logger.error(
'Invalid message format, expecting an s3ObjectKey in Message'
)
logger.error('Malformed Message: %r' % body_message)
continue
s3_log_files = message_json['s3ObjectKey']
for log_file in s3_log_files:
logger.debug('Downloading and parsing ' + log_file)
bucket = self.s3_connection.get_bucket(
message_json['s3Bucket'])
log_file_lookup = bucket.lookup(log_file)
events = self.process_file(log_file_lookup)
for event in events:
self.on_message(event)
self.taskQueue.delete_message(msg)
except KeyboardInterrupt:
sys.exit(1)
except Exception as e:
logger.exception(e)
time.sleep(3)
except (SSLEOFError, SSLError, socket.error) as e:
logger.info('Received network related error...reconnecting')
time.sleep(5)
self.connection, self.taskQueue = connect_sqs(
options.region, options.accesskey, options.secretkey,
options.taskexchange)
self.taskQueue.set_message_class(RawMessage)
def run(self):
while True:
try:
records = self.sqs_queue.receive_messages(
MaxNumberOfMessages=options.prefetch)
for msg in records:
body_message = msg.body
event = json.loads(body_message)
if not event["Message"]:
logger.error(
"Invalid message format for cloudtrail SQS messages"
)
logger.error("Malformed Message: %r" % body_message)
continue
if event["Message"] == "CloudTrail validation message.":
# We don't care about these messages
continue
message_json = json.loads(event["Message"])
if "s3ObjectKey" not in message_json:
logger.error(
"Invalid message format, expecting an s3ObjectKey in Message"
)
logger.error("Malformed Message: %r" % body_message)
continue
s3_log_files = message_json["s3ObjectKey"]
for log_file in s3_log_files:
logger.debug("Downloading and parsing " + log_file)
s3_obj = self.s3_client.get_object(
Bucket=message_json["s3Bucket"], Key=log_file)
events = self.parse_s3_file(s3_obj)
for event in events:
self.on_message(event)
msg.delete()
except (SSLEOFError, SSLError, socket.error):
logger.info("Received network related error...reconnecting")
time.sleep(5)
self.sqs_queue = connect_sqs(
region_name=options.region,
aws_access_key_id=options.accesskey,
aws_secret_access_key=options.secretkey,
task_exchange=options.taskexchange,
)
time.sleep(options.sleep_time)
def main():
if hasUWSGI:
logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id()))
else:
logger.info('started without uwsgi')
if options.mqprotocol not in ('sqs'):
logger.error('Can only process SQS queues, terminating')
sys.exit(1)
sqs_queue = connect_sqs(region_name=options.region,
aws_access_key_id=options.accesskey,
aws_secret_access_key=options.secretkey,
task_exchange=options.taskexchange)
# consume our queue
taskConsumer(sqs_queue, es, options).run()
def main():
if hasUWSGI:
logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id()))
else:
logger.info('started without uwsgi')
if options.mqprotocol not in ('sqs'):
logger.error('Can only process SQS queues, terminating')
sys.exit(1)
mqConn, eventTaskQueue = connect_sqs(options.region, options.accesskey,
options.secretkey,
options.taskexchange)
# consume our queue
taskConsumer(mqConn, eventTaskQueue, es, options).run()
def run(self):
self.taskQueue.set_message_class(RawMessage)
while True:
try:
records = self.taskQueue.get_messages(options.prefetch)
for msg in records:
body_message = msg.get_body()
event = json.loads(body_message)
if not event['Message']:
logger.error('Invalid message format for cloudtrail SQS messages')
logger.error('Malformed Message: %r' % body_message)
continue
if event['Message'] == 'CloudTrail validation message.':
# We don't care about these messages
continue
message_json = json.loads(event['Message'])
if 's3ObjectKey' not in message_json:
logger.error('Invalid message format, expecting an s3ObjectKey in Message')
logger.error('Malformed Message: %r' % body_message)
continue
s3_log_files = message_json['s3ObjectKey']
for log_file in s3_log_files:
logger.debug('Downloading and parsing ' + log_file)
bucket = self.s3_connection.get_bucket(message_json['s3Bucket'])
log_file_lookup = bucket.lookup(log_file)
events = self.process_file(log_file_lookup)
for event in events:
self.on_message(event)
self.taskQueue.delete_message(msg)
except (SSLEOFError, SSLError, socket.error):
logger.info('Received network related error...reconnecting')
time.sleep(5)
self.connection, self.taskQueue = connect_sqs(
task_exchange=options.taskexchange,
**get_aws_credentials(
options.region,
options.accesskey,
options.secretkey))
self.taskQueue.set_message_class(RawMessage)
def main():
if hasUWSGI:
logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id()))
else:
logger.info('started without uwsgi')
if options.mqprotocol not in ('sqs'):
logger.error('Can only process SQS queues, terminating')
sys.exit(1)
sqs_conn, eventTaskQueue = connect_sqs(
task_exchange=options.taskexchange,
**get_aws_credentials(
options.region,
options.accesskey,
options.secretkey))
# consume our queue
taskConsumer(sqs_conn, eventTaskQueue, es, options).run()
def main():
# meant only to talk to SQS using boto
# and process events as json.
if hasUWSGI:
logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id()))
else:
logger.info('started without uwsgi')
if options.mqprotocol not in ('sqs'):
logger.error('Can only process SQS queues, terminating')
sys.exit(1)
sqs_conn, eventTaskQueue = connect_sqs(
task_exchange=options.taskexchange,
**get_aws_credentials(options.region, options.accesskey,
options.secretkey))
# consume our queue
taskConsumer(sqs_conn, eventTaskQueue, es).run()
def main():
# meant only to talk to SQS using boto
# and process events as json.
if hasUWSGI:
logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id()))
else:
logger.info("started without uwsgi")
if options.mqprotocol not in ("sqs"):
logger.error("Can only process SQS queues, terminating")
sys.exit(1)
sqs_queue = connect_sqs(
region_name=options.region,
aws_access_key_id=options.accesskey,
aws_secret_access_key=options.secretkey,
task_exchange=options.taskexchange,
)
# consume our queue
taskConsumer(sqs_queue, es).run()
def run(self):
while True:
try:
records = self.sqs_queue.receive_messages(
MaxNumberOfMessages=options.prefetch)
for msg in records:
# msg.id is the id,
# get_body() should be json
# pre process the message a bit
tmp = msg.body
try:
msgbody = json.loads(tmp)
except ValueError:
# If Boto wrote to the queue, it might be base64 encoded, so let's decode that
try:
tmp = base64.b64decode(tmp)
msgbody = json.loads(tmp)
except Exception as e:
logger.error(
'Invalid message, not JSON <dropping message and continuing>: %r'
% msg.get_body())
msg.delete()
continue
# If this is still not a dict,
# let's just drop the message and move on
if type(msgbody) is not dict:
logger.debug(
"Message is not a dictionary, dropping message.")
msg.delete()
continue
event = dict()
event = msgbody
# Was this message sent by fluentd-sqs
fluentd_sqs_specific_fields = {
'az', 'instance_id', '__tag'
}
if fluentd_sqs_specific_fields.issubset(set(
msgbody.keys())):
# Until we can influence fluentd-sqs to set the
# 'customendpoint' key before submitting to SQS, we'll
# need to do it here
# TODO : Change nubis fluentd output to include
# 'customendpoint'
event['customendpoint'] = True
if 'tags' in event:
event['tags'].extend([options.taskexchange])
else:
event['tags'] = [options.taskexchange]
# process message
self.on_message(event, msg)
# delete message from queue
msg.delete()
time.sleep(options.sleep_time)
except ValueError as e:
logger.exception('Exception while handling message: %r' % e)
msg.delete()
except (SSLEOFError, SSLError, socket.error):
logger.info('Received network related error...reconnecting')
time.sleep(5)
self.sqs_queue = connect_sqs(options.region, options.accesskey,
options.secretkey,
options.taskexchange)
def run(self):
# Boto expects base64 encoded messages - but if the writer is not boto it's not necessarily base64 encoded
# Thus we've to detect that and decode or not decode accordingly
self.taskQueue.set_message_class(RawMessage)
while True:
try:
records = self.taskQueue.get_messages(options.prefetch)
for msg in records:
# msg.id is the id,
# get_body() should be json
# pre process the message a bit
tmp = msg.get_body()
try:
msgbody = json.loads(tmp)
except ValueError:
# If Boto wrote to the queue, it might be base64 encoded, so let's decode that
try:
tmp = base64.b64decode(tmp)
msgbody = json.loads(tmp)
except Exception as e:
logger.error('Invalid message, not JSON <dropping message and continuing>: %r' % msg.get_body())
self.taskQueue.delete_message(msg)
continue
# If this is still not a dict,
# let's just drop the message and move on
if type(msgbody) is not dict:
logger.debug("Message is not a dictionary, dropping message.")
self.taskQueue.delete_message(msg)
continue
event = dict()
event = msgbody
# Was this message sent by fluentd-sqs
fluentd_sqs_specific_fields = {
'az', 'instance_id', '__tag'}
if fluentd_sqs_specific_fields.issubset(
set(msgbody.keys())):
# Until we can influence fluentd-sqs to set the
# 'customendpoint' key before submitting to SQS, we'll
# need to do it here
# TODO : Change nubis fluentd output to include
# 'customendpoint'
event['customendpoint'] = True
if 'tags' in event:
event['tags'].extend([options.taskexchange])
else:
event['tags'] = [options.taskexchange]
# process message
self.on_message(event, msg)
# delete message from queue
self.taskQueue.delete_message(msg)
time.sleep(.1)
except ValueError as e:
logger.exception('Exception while handling message: %r' % e)
self.taskQueue.delete_message(msg)
except (SSLEOFError, SSLError, socket.error):
logger.info('Received network related error...reconnecting')
time.sleep(5)
self.connection, self.taskQueue = connect_sqs(
options.region,
options.accesskey,
options.secretkey,
options.taskexchange
)
self.taskQueue.set_message_class(RawMessage)
def run(self):
# Boto expects base64 encoded messages - but if the writer is not boto it's not necessarily base64 encoded
# Thus we've to detect that and decode or not decode accordingly
self.taskQueue.set_message_class(RawMessage)
while True:
try:
records = self.taskQueue.get_messages(options.prefetch)
for msg in records:
# msg.id is the id,
# get_body() should be json
# pre process the message a bit
tmp = msg.get_body()
try:
msgbody = json.loads(tmp)
except ValueError:
# If Boto wrote to the queue, it might be base64 encoded, so let's decode that
try:
tmp = base64.b64decode(tmp)
msgbody = json.loads(tmp)
except Exception as e:
logger.error(
'Invalid message, not JSON <dropping message and continuing>: %r'
% msg.get_body())
self.taskQueue.delete_message(msg)
continue
event = dict()
event = msgbody
# Was this message sent by fluentd-sqs
fluentd_sqs_specific_fields = {
'az', 'instance_id', '__tag'
}
if fluentd_sqs_specific_fields.issubset(set(
msgbody.keys())):
# Until we can influence fluentd-sqs to set the
# 'customendpoint' key before submitting to SQS, we'll
# need to do it here
# TODO : Change nubis fluentd output to include
# 'customendpoint'
event['customendpoint'] = True
if 'tags' in event:
event['tags'].extend([options.taskexchange])
else:
event['tags'] = [options.taskexchange]
# process message
self.on_message(event, msg)
# delete message from queue
self.taskQueue.delete_message(msg)
time.sleep(.1)
except ValueError as e:
logger.exception('Exception while handling message: %r' % e)
self.taskQueue.delete_message(msg)
except (SSLEOFError, SSLError, socket.error):
logger.info('Received network related error...reconnecting')
time.sleep(5)
self.connection, self.taskQueue = connect_sqs(
options.region, options.accesskey, options.secretkey,
options.taskexchange)
self.taskQueue.set_message_class(RawMessage)
