def process (self): try: reticLog.logInfo(self.logList, "( " + self.name + " ) Message processed on sink : " + self.name) return 0 except: errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0] reticLog.logError(self.logList, "( " + self.name + " ) Error during message processing in sink : " + self.name) reticLog.logError(self.logList, "( " + self.name + " ) " + errorMessage) return 1
def getMsg (self, message): try: reticLog.logInfo(self.logList, "( " + self.name + " ) Retrieving message for sink : " + self.name) # Re-initialize msg to get new message self.msg = '' self.msg = message reticLog.logInfo(self.logList, "( " + self.name + " ) Message retrieved in sink : " + self.name) return 0 except: errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0] reticLog.logError(self.logList, "( " + self.name + " ) Error during message retrieval in sink : " + self.name) reticLog.logError(self.logList, "( " + self.name + " ) " + errorMessage) return 1
def getRecordList(self): # Extraction of the fields and values to map to the SQL statement. # The method returns a list of dictionnaries reticLog.logInfo(self.logList, "( " + self.name + " ) getRecordList Start") msg = StringIO.StringIO() msg.write(self.msg) #print raw_input('msg here...') #print msg msg.seek(0) recordList = [] msgFormat = '' xmlRoot = None if self.metadata.has_key(msgFormat) : if self.metadata['msgFormat'] == 'xml': xmlRoot = ElementTree(msg) msgFormat = 'xml' else: msgFormat = 'flat' if msgFormat == '': try: xmlRoot = ElementTree(file=msg) msgFormat = 'xml' except: msgFormat = 'flat' reticLog.logDebug(self.logList, "Input format detected : " + msgFormat) # I am here if msgFormat == 'xml': recNb = 0 #Create an iterator iter = xmlRoot.getiterator() #traverse the xml tree for element in iter: if element.getchildren(): for child in element.getchildren(): prepRecord = {} if child.getchildren(): for subChild in child.getchildren(): if subChild.text == '-999': continue prepRecord[subChild.tag] = subChild.text recordList.append(prepRecord) reticLog.logDebug(self.logList, "All records processed.") #here for processing flat file elif msgFormat == 'flat': raise Exception('Do not support flat file at this time') reticLog.logInfo(self.logList, "( " + self.name + " ) getRecordList End") return recordList
def prepareUpdateObject(self, recordList,processInfo,firstTimeImport=True): reticLog.logInfo(self.logList, "( " + self.name + " ) prepareUpdateObject Start") #object list, to be add to session/real database objects = [] for record in recordList: result = processInfo(record) for ob in result: objects.append(ob) #objects.append(tempSQL) #reticLog.logDebug(self.logList, "SQL generated : " + tempSQL) reticLog.logInfo(self.logList, "( " + self.name + " ) prepareUpdateObject End") #this is for TWDB data, twdb specific #Initialize the first ValueID if firstTimeImport: objects[0].ValueID = 0 else: #get the max id column = testTWDBmodel.DataValues.ValueID.property.columns[0] seq = column.sequence maxid = self.session.query(func.max(column)) objects[0].ValueID = maxid + 1 return objects
def updateDB(self, objects): reticLog.logInfo(self.logList, "( " + self.name + " ) Starting update objects from sink : " + self.name) count = 0 for ob in objects : # print "Executing SQL : " + statement retries = self.retries execOk = 0 if count > 0: ob.ValueID = self.getMaxId() while retries >= 0 and execOk == 0: try: ############ self.session.add(ob) self.session.flush() print "recordNo == >", ob.ValueID, "generated" count = count+1 execOk = 1 #this is the handler for some violation of unique constriant on keys except exc.OperationalError: print "DB constraint violation happen" self.session.rollback() continue #execOk = 0 # retries = retries - 1 #this is the handler ofr invalid request error except exc.InvalidRequestError: print "DB constraint violation happen" self.session.rollback() continue #raise if execOk == 0 and retries < 0: raise "Database Exception : all retries failed" elif execOk == 1: # reticLog.logDebug(self.logList, "SQL executed : " + statement) try: self.session.commit() ## #self.session.close() #reticLog.logInfo(self.logList, "( " + self.name + " ) Update commited") except: self.session.rollback() #self.session.close() reticLog.logWarning(self.logList, "Commit Failed in SQLSink") else: errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0] reticLog.logWarning(self.logList, "Database Update failed : " + errorMessage) reticLog.logInfo(self.logList, "( " + self.name + " ) Number DB record (%d) added : " % count + self.name) reticLog.logInfo(self.logList, "( " + self.name + " ) Update of Databases ended in sink : " + self.name)