def __setstate__(self, state):
Component.__setstate__(self, state)
self.msg_queue = state['msg_queue']
self.connected = False
self.decoder = LogMessageDecoder()
self.clearing_overflow = False
self.overflow_filename = None
self.overflow_file = None
self.max_queued = int(get_cdbwriter_config('max_queued_msgs', '-1'))
if self.max_queued <= 0:
logger.info("message queue set to unlimited.")
self.max_queued = None
else:
self.overflow_filename = get_cdbwriter_config(
'overflow_file', None)
if self.max_queued and (self.overflow_filename == None):
logger.warning(
"No file given to catch maximum messages. Setting queue size to unlimited."
)
self.max_queued = None
if state.has_key('overflow') and self.max_queued:
self.overflow = state['overflow']
else:
self.overflow = False
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.sync_state = Cobalt.Util.FailureMode("Foreign Data Sync")
self.connected = False
self.msg_queue = []
self.decoder = LogMessageDecoder()
self.overflow = False
self.overflow_filename = None
self.overflow_file = None
self.clearing_overflow = False
self.max_queued = int(get_cdbwriter_config('max_queued_msgs', '-1'))
if self.max_queued <= 0:
logger.info("message queue set to unlimited.")
self.max_queued = None
self.overflow_filename = get_cdbwriter_config(
'overflow_file', None)
if self.overflow_filename == None:
logger.warning(
"No file given to catch maximum messages. Setting queue size to unlimited."
)
self.max_queued = None
def __setstate__(self, state):
Component.__setstate__(self, state)
self.msg_queue = state['msg_queue']
self.connected = False
self.decoder = LogMessageDecoder()
self.clearing_overflow = False
self.overflow_filename = None
self.overflow_file = None
self.max_queued = int(get_cdbwriter_config('max_queued_msgs', '-1'))
if self.max_queued <= 0:
logger.info("message queue set to unlimited.")
self.max_queued = None
else:
self.overflow_filename = get_cdbwriter_config('overflow_file', None)
if self.max_queued and (self.overflow_filename == None):
logger.warning("No file given to catch maximum messages. Setting queue size to unlimited.")
self.max_queued = None
if state.has_key('overflow') and self.max_queued:
self.overflow = state['overflow']
else:
self.overflow = False
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.sync_state = Cobalt.Util.FailureMode("Foreign Data Sync")
self.connected = False
self.msg_queue = []
self.decoder = LogMessageDecoder()
self.overflow = False
self.overflow_filename = None
self.overflow_file = None
self.clearing_overflow = False
self.max_queued = int(get_cdbwriter_config('max_queued_msgs', '-1'))
if self.max_queued <= 0:
logger.info("message queue set to unlimited.")
self.max_queued = None
self.overflow_filename = get_cdbwriter_config('overflow_file', None)
if self.overflow_filename == None:
logger.warning("No file given to catch maximum messages. Setting queue size to unlimited.")
self.max_queued = None
class MessageQueue(Component):
name = "cdbwriter"
implementation = "cdbwriter"
logger = logging.getLogger("Cobalt.Components.cdbwriter")
_configfields = ['user', 'pwd', 'database', 'schema']
_config = ConfigParser.ConfigParser()
_config.read(Cobalt.CONFIG_FILES)
if not config._sections.has_key('cdbwriter'):
logger.error('"cdbwriter" section missing from config file.')
config = _config._sections['cdbwriter']
mfields = [field for field in _configfields if not config.has_key(field)]
if mfields:
logger.error("Missing option(s) in cobalt config file [cdbwriter] section: %s" % (" ".join(mfields)))
sys.exit(1)
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.sync_state = Cobalt.Util.FailureMode("Foreign Data Sync")
self.connected = False
self.msg_queue = []
self.decoder = LogMessageDecoder()
self.overflow = False
self.overflow_filename = None
self.overflow_file = None
self.clearing_overflow = False
self.max_queued = int(get_cdbwriter_config('max_queued_msgs', '-1'))
if self.max_queued <= 0:
logger.info("message queue set to unlimited.")
self.max_queued = None
self.overflow_filename = get_cdbwriter_config('overflow_file', None)
if self.overflow_filename == None:
logger.warning("No file given to catch maximum messages. Setting queue size to unlimited.")
self.max_queued = None
def __getstate__(self):
state = {}
state.update(Component.__getstate__(self))
state.update({
'cdbwriter_version': 1,
'msg_queue': self.msg_queue,
'overflow': self.overflow})
return state
def __setstate__(self, state):
Component.__setstate__(self, state)
self.msg_queue = state['msg_queue']
self.connected = False
self.decoder = LogMessageDecoder()
self.clearing_overflow = False
self.overflow_filename = None
self.overflow_file = None
self.max_queued = int(get_cdbwriter_config('max_queued_msgs', '-1'))
if self.max_queued <= 0:
logger.info("message queue set to unlimited.")
self.max_queued = None
else:
self.overflow_filename = get_cdbwriter_config('overflow_file', None)
if self.max_queued and (self.overflow_filename == None):
logger.warning("No file given to catch maximum messages. Setting queue size to unlimited.")
self.max_queued = None
if state.has_key('overflow') and self.max_queued:
self.overflow = state['overflow']
else:
self.overflow = False
def init_database_connection(self):
user = get_cdbwriter_config('user', None)
pwd = get_cdbwriter_config('pwd', None)
database = get_cdbwriter_config('database', None)
schema = get_cdbwriter_config('schema', None)
try:
self.database_writer = DatabaseWriter(database, user, pwd, schema)
except:
#make this a log statement
logging.error("Unable to connect to %s as %s" % (database, user))
self.connected = False
logging.debug(traceback.format_exc())
else:
self.connected = True
def iterate(self):
"""Go through the messages that are sitting on the queue and
load them into the database."""
#if we're not connected, try to reconnect to the database
if not self.connected:
logger.debug("Attempting reconnection.")
self.init_database_connection()
if self.connected and self.overflow:
self.clearing_overflow = True
self.open_overflow('r')
if self.overflow_file:
overflow_queue = [self.decoder.decode(line)
for line in self.overflow_file]
overflow_queue.extend(self.msg_queue)
self.msg_queue = overflow_queue
self.close_overflow()
self.del_overflow()
self.overflow = False
while self.msg_queue and self.connected:
msg = self.msg_queue[0]
try:
self.database_writer.addMessage(msg)
except db2util.adapterError:
logger.error ("Error updating databse. Unable to add message due to adapter error. Message dropped.")
logging.debug(traceback.format_exc())
self.msg_queue.pop(0)
except:
logger.error ("Error updating databse. Unable to add message.")
logging.debug(traceback.format_exc())
self.connected = False
#if we were clearing an overflow, here we go again.
if ((self.max_queued != None) and
(len(self.msg_queue) >= self.max_queued)):
self.overflow = True
self.open_overflow('a')
if self.overflow_file != None:
self.queue_to_overflow()
self.close_overflow()
break
else:
#message added
self.msg_queue.pop(0)
self.clearing_overflow = False
iterate = automatic(iterate)
def add_message(self, msg):
#keep the queue from consuming all memory
if ((self.max_queued != None) and
(len(self.msg_queue) >= self.max_queued)and
(not self.clearing_overflow)):
self.overflow = True
self.open_overflow('a')
if self.overflow_file == None:
logger.critical("MESSAGE DROPPED: %s", msg)
else:
self.queue_to_overflow()
self.close_overflow()
#and now queue as normal
msgDict = None
try:
msgDict = self.decoder.decode(msg)
except ValueError:
logger.error("Bad message recieved. Failed to decode string %s" % msg)
return
except:
logging.debug(traceback.format_exc())
self.msg_queue.append(msgDict)
add_message = exposed(add_message)
def save_me(self):
Component.save(self)
save_me = automatic(save_me)
def open_overflow(self, mode):
try:
self.overflow_file = open(self.overflow_filename,mode)
except:
self.logger.critical("Unable to open overflow file! Information to database will be lost!")
def close_overflow(self):
if self.overflow and self.overflow_file:
self.overflow_file.close()
self.overflow_file = None
def del_overflow(self):
os.remove(self.overflow_filename)
def queue_to_overflow(self):
elements_written = 0
if len(self.msg_queue) == 0:
return
while self.msg_queue:
msg = self.msg_queue.pop(0)
try:
self.overflow_file.write(json.dumps(msg, cls=LogMessageEncoder)+'\n')
elements_written += 1
except IOError:
logger.error('Could only partially empty queue, %d messages written' %
elements_written)
self.msg_queue.insert(0, msg)
if elements_written > 0:
del self.msg_queue[0:elements_written-1] #empty the queue of what we have written
return len(self.msg_queue)
class MessageQueue(Component):
name = "cdbwriter"
implementation = "cdbwriter"
logger = logging.getLogger("Cobalt.Components.cdbwriter")
_configfields = ['user', 'pwd', 'database', 'schema']
_config = ConfigParser.ConfigParser()
_config.read(Cobalt.CONFIG_FILES)
if not config._sections.has_key('cdbwriter'):
logger.error('"cdbwriter" section missing from config file.')
config = _config._sections['cdbwriter']
mfields = [field for field in _configfields if not config.has_key(field)]
if mfields:
logger.error(
"Missing option(s) in cobalt config file [cdbwriter] section: %s" %
(" ".join(mfields)))
sys.exit(1)
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.sync_state = Cobalt.Util.FailureMode("Foreign Data Sync")
self.connected = False
self.msg_queue = []
self.decoder = LogMessageDecoder()
self.overflow = False
self.overflow_filename = None
self.overflow_file = None
self.clearing_overflow = False
self.max_queued = int(get_cdbwriter_config('max_queued_msgs', '-1'))
if self.max_queued <= 0:
logger.info("message queue set to unlimited.")
self.max_queued = None
self.overflow_filename = get_cdbwriter_config(
'overflow_file', None)
if self.overflow_filename == None:
logger.warning(
"No file given to catch maximum messages. Setting queue size to unlimited."
)
self.max_queued = None
def __getstate__(self):
state = {}
state.update(Component.__getstate__(self))
state.update({
'cdbwriter_version': 1,
'msg_queue': self.msg_queue,
'overflow': self.overflow
})
return state
def __setstate__(self, state):
Component.__setstate__(self, state)
self.msg_queue = state['msg_queue']
self.connected = False
self.decoder = LogMessageDecoder()
self.clearing_overflow = False
self.overflow_filename = None
self.overflow_file = None
self.max_queued = int(get_cdbwriter_config('max_queued_msgs', '-1'))
if self.max_queued <= 0:
logger.info("message queue set to unlimited.")
self.max_queued = None
else:
self.overflow_filename = get_cdbwriter_config(
'overflow_file', None)
if self.max_queued and (self.overflow_filename == None):
logger.warning(
"No file given to catch maximum messages. Setting queue size to unlimited."
)
self.max_queued = None
if state.has_key('overflow') and self.max_queued:
self.overflow = state['overflow']
else:
self.overflow = False
def init_database_connection(self):
user = get_cdbwriter_config('user', None)
pwd = get_cdbwriter_config('pwd', None)
database = get_cdbwriter_config('database', None)
schema = get_cdbwriter_config('schema', None)
try:
self.database_writer = DatabaseWriter(database, user, pwd, schema)
except:
#make this a log statement
logging.error("Unable to connect to %s as %s" % (database, user))
self.connected = False
logging.debug(traceback.format_exc())
else:
self.connected = True
def iterate(self):
"""Go through the messages that are sitting on the queue and
load them into the database."""
#if we're not connected, try to reconnect to the database
if not self.connected:
logger.debug("Attempting reconnection.")
self.init_database_connection()
if self.connected and self.overflow:
self.clearing_overflow = True
self.open_overflow('r')
if self.overflow_file:
overflow_queue = [
self.decoder.decode(line) for line in self.overflow_file
]
overflow_queue.extend(self.msg_queue)
self.msg_queue = overflow_queue
self.close_overflow()
self.del_overflow()
self.overflow = False
while self.msg_queue and self.connected:
msg = self.msg_queue[0]
try:
self.database_writer.addMessage(msg)
except db2util.adapterError:
logger.error(
"Error updating databse. Unable to add message due to adapter error. Message dropped."
)
logging.debug(traceback.format_exc())
self.msg_queue.pop(0)
except:
logger.error("Error updating databse. Unable to add message.")
logging.debug(traceback.format_exc())
self.connected = False
#if we were clearing an overflow, here we go again.
if ((self.max_queued != None)
and (len(self.msg_queue) >= self.max_queued)):
self.overflow = True
self.open_overflow('a')
if self.overflow_file != None:
self.queue_to_overflow()
self.close_overflow()
break
else:
#message added
self.msg_queue.pop(0)
self.clearing_overflow = False
iterate = automatic(iterate)
def add_message(self, msg):
#keep the queue from consuming all memory
if ((self.max_queued != None)
and (len(self.msg_queue) >= self.max_queued)
and (not self.clearing_overflow)):
self.overflow = True
self.open_overflow('a')
if self.overflow_file == None:
logger.critical("MESSAGE DROPPED: %s", msg)
else:
self.queue_to_overflow()
self.close_overflow()
#and now queue as normal
msgDict = None
try:
msgDict = self.decoder.decode(msg)
except ValueError:
logger.error("Bad message recieved. Failed to decode string %s" %
msg)
return
except:
logging.debug(traceback.format_exc())
self.msg_queue.append(msgDict)
add_message = exposed(add_message)
def save_me(self):
Component.save(self)
save_me = automatic(save_me)
def open_overflow(self, mode):
try:
self.overflow_file = open(self.overflow_filename, mode)
except:
self.logger.critical(
"Unable to open overflow file! Information to database will be lost!"
)
def close_overflow(self):
if self.overflow and self.overflow_file:
self.overflow_file.close()
self.overflow_file = None
def del_overflow(self):
os.remove(self.overflow_filename)
def queue_to_overflow(self):
elements_written = 0
if len(self.msg_queue) == 0:
return
while self.msg_queue:
msg = self.msg_queue.pop(0)
try:
self.overflow_file.write(
json.dumps(msg, cls=LogMessageEncoder) + '\n')
elements_written += 1
except IOError:
logger.error(
'Could only partially empty queue, %d messages written' %
elements_written)
self.msg_queue.insert(0, msg)
if elements_written > 0:
del self.msg_queue[0:elements_written -
1] #empty the queue of what we have written
return len(self.msg_queue)