def run(self): while True: try: records = self.sqs_queue.receive_messages( MaxNumberOfMessages=options.prefetch) for msg in records: msg_body = msg.body try: # get_body() should be json message_json = json.loads(msg_body) self.on_message(message_json) # delete message from queue msg.delete() except ValueError: logger.error( 'Invalid message, not JSON <dropping message and continuing>: %r' % msg_body) msg.delete() continue time.sleep(options.sleep_time) except (SSLEOFError, SSLError, socket.error): logger.info('Received network related error...reconnecting') time.sleep(5) self.sqs_queue = connect_sqs(options.region, options.accesskey, options.secretkey, options.taskexchange)
def run(self): self.taskQueue.set_message_class(RawMessage) while True: try: records = self.taskQueue.get_messages(self.options.prefetch) for msg in records: msg_body = msg.get_body() try: # get_body() should be json message_json = json.loads(msg_body) self.on_message(message_json) # delete message from queue self.taskQueue.delete_message(msg) except ValueError: logger.error('Invalid message, not JSON <dropping message and continuing>: %r' % msg_body) self.taskQueue.delete_message(msg) continue time.sleep(.1) except (SSLEOFError, SSLError, socket.error): logger.info('Received network related error...reconnecting') time.sleep(5) self.connection, self.taskQueue = connect_sqs( options.region, options.accesskey, options.secretkey, options.taskexchange ) self.taskQueue.set_message_class(RawMessage)
def run(self): while True: try: records = self.sqs_queue.receive_messages( MaxNumberOfMessages=options.prefetch) for msg in records: # msg.id is the id, # get_body() should be json # pre process the message a bit tmp = msg.body try: msgbody = json.loads(tmp) except ValueError: # If Boto wrote to the queue, it might be base64 encoded, so let's decode that try: tmp = base64.b64decode(tmp) msgbody = json.loads(tmp) except Exception as e: logger.error( "Invalid message, not JSON <dropping message and continuing>: %r" % msg.get_body()) msg.delete() continue # If this is still not a dict, # let's just drop the message and move on if type(msgbody) is not dict: logger.debug( "Message is not a dictionary, dropping message.") msg.delete() continue event = dict() event = msgbody if "tags" in event: event["tags"].extend([options.taskexchange]) else: event["tags"] = [options.taskexchange] # process message self.on_message(event, msg) # delete message from queue msg.delete() time.sleep(options.sleep_time) except ValueError as e: logger.exception("Exception while handling message: %r" % e) msg.delete() except (SSLEOFError, SSLError, socket.error): logger.info("Received network related error...reconnecting") time.sleep(5) self.sqs_queue = connect_sqs(options.region, options.accesskey, options.secretkey, options.taskexchange)
def run(self): self.taskQueue.set_message_class(RawMessage) while True: try: records = self.taskQueue.get_messages(options.prefetch) for msg in records: body_message = msg.get_body() event = json.loads(body_message) if not event['Message']: logger.error( 'Invalid message format for cloudtrail SQS messages' ) logger.error('Malformed Message: %r' % body_message) continue if event['Message'] == 'CloudTrail validation message.': # We don't care about these messages continue message_json = json.loads(event['Message']) if 's3ObjectKey' not in message_json.keys(): logger.error( 'Invalid message format, expecting an s3ObjectKey in Message' ) logger.error('Malformed Message: %r' % body_message) continue s3_log_files = message_json['s3ObjectKey'] for log_file in s3_log_files: logger.debug('Downloading and parsing ' + log_file) bucket = self.s3_connection.get_bucket( message_json['s3Bucket']) log_file_lookup = bucket.lookup(log_file) events = self.process_file(log_file_lookup) for event in events: self.on_message(event) self.taskQueue.delete_message(msg) except KeyboardInterrupt: sys.exit(1) except Exception as e: logger.exception(e) time.sleep(3) except (SSLEOFError, SSLError, socket.error) as e: logger.info('Received network related error...reconnecting') time.sleep(5) self.connection, self.taskQueue = connect_sqs( options.region, options.accesskey, options.secretkey, options.taskexchange) self.taskQueue.set_message_class(RawMessage)
def run(self): while True: try: records = self.sqs_queue.receive_messages( MaxNumberOfMessages=options.prefetch) for msg in records: body_message = msg.body event = json.loads(body_message) if not event["Message"]: logger.error( "Invalid message format for cloudtrail SQS messages" ) logger.error("Malformed Message: %r" % body_message) continue if event["Message"] == "CloudTrail validation message.": # We don't care about these messages continue message_json = json.loads(event["Message"]) if "s3ObjectKey" not in message_json: logger.error( "Invalid message format, expecting an s3ObjectKey in Message" ) logger.error("Malformed Message: %r" % body_message) continue s3_log_files = message_json["s3ObjectKey"] for log_file in s3_log_files: logger.debug("Downloading and parsing " + log_file) s3_obj = self.s3_client.get_object( Bucket=message_json["s3Bucket"], Key=log_file) events = self.parse_s3_file(s3_obj) for event in events: self.on_message(event) msg.delete() except (SSLEOFError, SSLError, socket.error): logger.info("Received network related error...reconnecting") time.sleep(5) self.sqs_queue = connect_sqs( region_name=options.region, aws_access_key_id=options.accesskey, aws_secret_access_key=options.secretkey, task_exchange=options.taskexchange, ) time.sleep(options.sleep_time)
def main(): if hasUWSGI: logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id())) else: logger.info('started without uwsgi') if options.mqprotocol not in ('sqs'): logger.error('Can only process SQS queues, terminating') sys.exit(1) sqs_queue = connect_sqs(region_name=options.region, aws_access_key_id=options.accesskey, aws_secret_access_key=options.secretkey, task_exchange=options.taskexchange) # consume our queue taskConsumer(sqs_queue, es, options).run()
def main(): if hasUWSGI: logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id())) else: logger.info('started without uwsgi') if options.mqprotocol not in ('sqs'): logger.error('Can only process SQS queues, terminating') sys.exit(1) mqConn, eventTaskQueue = connect_sqs(options.region, options.accesskey, options.secretkey, options.taskexchange) # consume our queue taskConsumer(mqConn, eventTaskQueue, es, options).run()
def run(self): self.taskQueue.set_message_class(RawMessage) while True: try: records = self.taskQueue.get_messages(options.prefetch) for msg in records: body_message = msg.get_body() event = json.loads(body_message) if not event['Message']: logger.error('Invalid message format for cloudtrail SQS messages') logger.error('Malformed Message: %r' % body_message) continue if event['Message'] == 'CloudTrail validation message.': # We don't care about these messages continue message_json = json.loads(event['Message']) if 's3ObjectKey' not in message_json: logger.error('Invalid message format, expecting an s3ObjectKey in Message') logger.error('Malformed Message: %r' % body_message) continue s3_log_files = message_json['s3ObjectKey'] for log_file in s3_log_files: logger.debug('Downloading and parsing ' + log_file) bucket = self.s3_connection.get_bucket(message_json['s3Bucket']) log_file_lookup = bucket.lookup(log_file) events = self.process_file(log_file_lookup) for event in events: self.on_message(event) self.taskQueue.delete_message(msg) except (SSLEOFError, SSLError, socket.error): logger.info('Received network related error...reconnecting') time.sleep(5) self.connection, self.taskQueue = connect_sqs( task_exchange=options.taskexchange, **get_aws_credentials( options.region, options.accesskey, options.secretkey)) self.taskQueue.set_message_class(RawMessage)
def main(): if hasUWSGI: logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id())) else: logger.info('started without uwsgi') if options.mqprotocol not in ('sqs'): logger.error('Can only process SQS queues, terminating') sys.exit(1) sqs_conn, eventTaskQueue = connect_sqs( task_exchange=options.taskexchange, **get_aws_credentials( options.region, options.accesskey, options.secretkey)) # consume our queue taskConsumer(sqs_conn, eventTaskQueue, es, options).run()
def main(): # meant only to talk to SQS using boto # and process events as json. if hasUWSGI: logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id())) else: logger.info('started without uwsgi') if options.mqprotocol not in ('sqs'): logger.error('Can only process SQS queues, terminating') sys.exit(1) sqs_conn, eventTaskQueue = connect_sqs( task_exchange=options.taskexchange, **get_aws_credentials(options.region, options.accesskey, options.secretkey)) # consume our queue taskConsumer(sqs_conn, eventTaskQueue, es).run()
def main(): # meant only to talk to SQS using boto # and process events as json. if hasUWSGI: logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id())) else: logger.info("started without uwsgi") if options.mqprotocol not in ("sqs"): logger.error("Can only process SQS queues, terminating") sys.exit(1) sqs_queue = connect_sqs( region_name=options.region, aws_access_key_id=options.accesskey, aws_secret_access_key=options.secretkey, task_exchange=options.taskexchange, ) # consume our queue taskConsumer(sqs_queue, es).run()
def run(self): while True: try: records = self.sqs_queue.receive_messages( MaxNumberOfMessages=options.prefetch) for msg in records: # msg.id is the id, # get_body() should be json # pre process the message a bit tmp = msg.body try: msgbody = json.loads(tmp) except ValueError: # If Boto wrote to the queue, it might be base64 encoded, so let's decode that try: tmp = base64.b64decode(tmp) msgbody = json.loads(tmp) except Exception as e: logger.error( 'Invalid message, not JSON <dropping message and continuing>: %r' % msg.get_body()) msg.delete() continue # If this is still not a dict, # let's just drop the message and move on if type(msgbody) is not dict: logger.debug( "Message is not a dictionary, dropping message.") msg.delete() continue event = dict() event = msgbody # Was this message sent by fluentd-sqs fluentd_sqs_specific_fields = { 'az', 'instance_id', '__tag' } if fluentd_sqs_specific_fields.issubset(set( msgbody.keys())): # Until we can influence fluentd-sqs to set the # 'customendpoint' key before submitting to SQS, we'll # need to do it here # TODO : Change nubis fluentd output to include # 'customendpoint' event['customendpoint'] = True if 'tags' in event: event['tags'].extend([options.taskexchange]) else: event['tags'] = [options.taskexchange] # process message self.on_message(event, msg) # delete message from queue msg.delete() time.sleep(options.sleep_time) except ValueError as e: logger.exception('Exception while handling message: %r' % e) msg.delete() except (SSLEOFError, SSLError, socket.error): logger.info('Received network related error...reconnecting') time.sleep(5) self.sqs_queue = connect_sqs(options.region, options.accesskey, options.secretkey, options.taskexchange)
def run(self): # Boto expects base64 encoded messages - but if the writer is not boto it's not necessarily base64 encoded # Thus we've to detect that and decode or not decode accordingly self.taskQueue.set_message_class(RawMessage) while True: try: records = self.taskQueue.get_messages(options.prefetch) for msg in records: # msg.id is the id, # get_body() should be json # pre process the message a bit tmp = msg.get_body() try: msgbody = json.loads(tmp) except ValueError: # If Boto wrote to the queue, it might be base64 encoded, so let's decode that try: tmp = base64.b64decode(tmp) msgbody = json.loads(tmp) except Exception as e: logger.error('Invalid message, not JSON <dropping message and continuing>: %r' % msg.get_body()) self.taskQueue.delete_message(msg) continue # If this is still not a dict, # let's just drop the message and move on if type(msgbody) is not dict: logger.debug("Message is not a dictionary, dropping message.") self.taskQueue.delete_message(msg) continue event = dict() event = msgbody # Was this message sent by fluentd-sqs fluentd_sqs_specific_fields = { 'az', 'instance_id', '__tag'} if fluentd_sqs_specific_fields.issubset( set(msgbody.keys())): # Until we can influence fluentd-sqs to set the # 'customendpoint' key before submitting to SQS, we'll # need to do it here # TODO : Change nubis fluentd output to include # 'customendpoint' event['customendpoint'] = True if 'tags' in event: event['tags'].extend([options.taskexchange]) else: event['tags'] = [options.taskexchange] # process message self.on_message(event, msg) # delete message from queue self.taskQueue.delete_message(msg) time.sleep(.1) except ValueError as e: logger.exception('Exception while handling message: %r' % e) self.taskQueue.delete_message(msg) except (SSLEOFError, SSLError, socket.error): logger.info('Received network related error...reconnecting') time.sleep(5) self.connection, self.taskQueue = connect_sqs( options.region, options.accesskey, options.secretkey, options.taskexchange ) self.taskQueue.set_message_class(RawMessage)
def run(self): # Boto expects base64 encoded messages - but if the writer is not boto it's not necessarily base64 encoded # Thus we've to detect that and decode or not decode accordingly self.taskQueue.set_message_class(RawMessage) while True: try: records = self.taskQueue.get_messages(options.prefetch) for msg in records: # msg.id is the id, # get_body() should be json # pre process the message a bit tmp = msg.get_body() try: msgbody = json.loads(tmp) except ValueError: # If Boto wrote to the queue, it might be base64 encoded, so let's decode that try: tmp = base64.b64decode(tmp) msgbody = json.loads(tmp) except Exception as e: logger.error( 'Invalid message, not JSON <dropping message and continuing>: %r' % msg.get_body()) self.taskQueue.delete_message(msg) continue event = dict() event = msgbody # Was this message sent by fluentd-sqs fluentd_sqs_specific_fields = { 'az', 'instance_id', '__tag' } if fluentd_sqs_specific_fields.issubset(set( msgbody.keys())): # Until we can influence fluentd-sqs to set the # 'customendpoint' key before submitting to SQS, we'll # need to do it here # TODO : Change nubis fluentd output to include # 'customendpoint' event['customendpoint'] = True if 'tags' in event: event['tags'].extend([options.taskexchange]) else: event['tags'] = [options.taskexchange] # process message self.on_message(event, msg) # delete message from queue self.taskQueue.delete_message(msg) time.sleep(.1) except ValueError as e: logger.exception('Exception while handling message: %r' % e) self.taskQueue.delete_message(msg) except (SSLEOFError, SSLError, socket.error): logger.info('Received network related error...reconnecting') time.sleep(5) self.connection, self.taskQueue = connect_sqs( options.region, options.accesskey, options.secretkey, options.taskexchange) self.taskQueue.set_message_class(RawMessage)