Esempio n. 1
0
 def __init__ (self, args, logger):
     try:
         self.args = args
         self.logList = logger
         self.name = args['name']
         self.SOAPMessage = args['SOAPMessage']
         self.SOAPAction = args['SOAPAction']
         self.url = args['url']
         self.proxyUrl = args['proxyUrl']
         self.params = {}
         for key in self.args.keys():
             if key[0] == '_':
                 self.params[key] = self.args[key]
         self.exitOnError = args['exitOnError']
         self.nbMsg = 0
         self.msg = ''
         self.msgName = ''            
         self.metadata = {}
         reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' +  "Source " + self.name + " initialized.")
     except KeyError:
         try:
             reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Error during init phase of source : " + self.name)
         except:
             reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Error during init phase of source")
         reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Parameter " + str(sys.exc_info()[1]) + " is missing in component definition" )
         sys.exit(1)
     except:
         errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
         reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Unknown error during initialization in source : " + self.name)
         reticLog.logError(self.logList, '( ' + self.name + ' ) ' + errorMessage)
         sys.exit(1)
Esempio n. 2
0
 def getMsg (self, message):
     try:
         reticLog.logInfo(self.logList, "( " + self.name + " ) Retrieving message for sink : " + self.name)
         # Re-initialize msg to get new message
         self.msg = ''
         self.msg = message
         return 0
     except:
         errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
         reticLog.logError(self.logList, "( " + self.name + " ) Error during message retrieval in sink : " + self.name)                        
         reticLog.logError(self.logList, "( " + self.name + " ) " + errorMessage)
         return 1
Esempio n. 3
0
 def getMsg (self, message):
     'Initializes input buffer with message content'
     try:
         reticLog.logInfo(self.logList, '(' + self.name + ') ' + "Getting message into sink")
         self.InMsg = ''
         self.msg = ''
         self.InMsg = message
         return 0
     except:
         errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
         reticLog.logError(self.logList, '(' + self.name + ') ' + "Error during message retrieval in sink" )                        
         reticLog.logError(self.logList, '(' + self.name + ') ' + errorMessage)
         return 1
Esempio n. 4
0
 def start (self):
     'Start the source of the adaptor (begin work...)'
     reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' +  "Starting the source adaptor")
     try:
         self.connect()
         self.getMsg()
         self.ftpConn.close()
         return 0
     except:
         errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
         reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Error on adaptor start on source : " + self.name)                        
         reticLog.logError(self.logList, '( ' + self.name + ' ) ' + errorMessage)
         if self.exitOnError.lower() == 'y':
             sys.exit(1)
Esempio n. 5
0
    def __init__ (self, args, logger):
        try:
            self.logList = logger
            self.name = args['name']
            self.args = args
            reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' +  "Intitializing SQLTreeSource : " + self.name)            
            self.exitOnError = 'y'
            self.xpath = XPathAPI()
            self.metadata = {}
            self.dsn = args['dsn']
            self.user = args['user']
            self.password = args['password']
            self.dbType = args['dbType']
#            self.db = args['db']
            self.msgSize = args['msgSize']
            self.treeQ = args['treeQ']
            self.rootTag = args['rootTag']
            self.encoding = args['encoding']
            self.msgSize = float(args['msgSize'])
            self.msgList = []
            self.fieldList = []
            self.nbMsg = 0
            self.tmpFileName = 'tmp_SQLTreeSource' + str(int(random.random()*100000000)) + '.dat'
#            self.msg = open(self.tmpFileName,'w+')
            self.msg = ''
            self.tmpMsg = StringIO.StringIO()
            self.msgName = ''
            self.curArgs = {}
            self.dbConnection = dbFactory.dbFactory(args, self.logList)
            self.dbConnection.connect(args)            
            if args.has_key('exitOnError'):
                self.exitOnError = args['exitOnError']
        except KeyError:
            reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Error on SQLTreeSource initialization")
            reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Parameter " + str(sys.exc_info()[1]) + " is missing on source definition" )
            sys.exit(1)
        except:
            errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
            reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Unknown error on SQLTreeSource initialization. Exiting...")
            reticLog.logError(self.logList, '( ' + self.name + ' ) ' + errorMessage)         
            sys.exit(1)                
Esempio n. 6
0
 def next(self):
     if self.nbMsg == 0:
         return 0
     else:
         try:
             reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' +  "Starting processing phase of source " + self.name)
             self.updateAttributesFromMetadata()
             print "updateMetTermine"
             SOAPMessage = self.SOAPMessage
             for key in self.params.keys():
                 print "key : " + key
                 SOAPMessage = string.replace(SOAPMessage,'**' + key[1:] + '**',self.params[key])
             print "SOAPMsg : "  + SOAPMessage
             reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' +  "Executing SOAP request")
             reticLog.logDebug(self.logList, '( ' + self.name + ' ) ' +  "SOAP Request : " + SOAPMessage)
             if self.proxyUrl <> '':
                 urlopener = urllib.FancyURLopener( {'http' : self.proxyUrl } )
             else:
                 urlopener = urllib.URLopener()
             urlopener.addheaders.append(('Content-Length', str(len(SOAPMessage))))
             urlopener.addheaders.append(('Content-type', 'text/xml; charset="utf-8"'))
             urlopener.addheaders.append(('SOAPAction', self.SOAPAction))
             
             resp = urlopener.open(self.url, SOAPMessage)
             response = resp.read()
             reticLog.logDebug(self.logList, '( ' + self.name + ' ) ' +  "SOAP Response : " + response)
             self.msg = response
 
             reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' +  "source " + self.name + " executed ok")
             return 1
         except:
             errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
             reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Error during process phase of source : " + self.name)
             reticLog.logError(self.logList, '( ' + self.name + ' ) ' + errorMessage)
             return 0                    
Esempio n. 7
0
 def commit(self):
     'Commit the current message treatment'
     if self.nbMsg > 0 : 
         reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' +  "Commiting msg " + self.msgList[0] + " on source : " + self.msgName)
         msgName = self.msgList[0]
         try:    
             if self.newExtension != '':
                 reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' +  "Processing file : " + self.msgList[0])
                 self.connect()
                 self.ftpConn.sendcmd('RNFR ' + self.msgName)
                 self.ftpConn.sendcmd('RNTO ' + self.msgName + '.' + self.newExtension)
             self.nbMsg = self.nbMsg - 1 
             self.msgList = self.msgList[1:]
             self.msg = ''
             return 1
         except:
             errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
             reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Error on commit phase on source - File : " + msgName)                        
             reticLog.logError(self.logList, '( ' + self.name + ' ) ' + errorMessage)
             if self.exitOnError.lower() == 'y':
                 sys.exit(1)
             return 0
     else:
         return 0
Esempio n. 8
0
    def next(self):
        'Get the next message to be processed or return that sources are dry'
        try:
            self.connect()
        except:
            errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
            reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Error on message retrieval on source : " + self.name)                        
            reticLog.logError(self.logList, '( ' + self.name + ' ) ' + errorMessage)
            if self.exitOnError.lower() == 'y':
                sys.exit(1)
                
        if self.nbMsg == 0:
            reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' +  "Sources dry on source ")
            return 0
        else:
            try:
                reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' +  "Messages Left on queue of adaptor : " + str(self.nbMsg))
                reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' +  "Processing file : " + self.msgList[0])
                self.msgName = self.msgList[0]

                dotIndex = string.find(self.msgName,'.')
                if dotIndex > 0:
                    self.metadata['filename'] = string.split(self.msgName,'.')[0]
                    self.metadata['extension'] = string.split(self.msgName,'.')[1]
                else:
                    self.metadata['filename'] = self.msgName
                    self.metadata['extension'] = ''
                
                msg = StringIO.StringIO()
                sys.stdout = msg
                self.ftpConn.retrlines('RETR ' + self.msgName)
                sys.stdout = sys.__stdout__
                msg.seek(0)
                self.msg = msg.read()
                self.ftpConn.close()
                return 1
            except:
                errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
                reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Error on message retrieval on source : " + self.name)                        
                reticLog.logError(self.logList, '( ' + self.name + ' ) ' + errorMessage)
                if self.exitOnError.lower() == 'y':
                    sys.exit(1)
                return 1
Esempio n. 9
0
 def __init__ (self, args, logger):
     try:
         self.logList = logger
         self.name = args['name']
         self.exitOnError = 'y'
         reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' +  "Intitializing ftpSource : " + self.name)            
         self.ftpHost = args['ftpHost']
         self.ftpPort = int(21)
         self.ftpUser = args['ftpUser']
         self.ftpPass = args['ftpPass']
         self.filePath = args['filePath']
         self.fileFilter = args['fileFilter']
         self.newExtension = args['newExtension']
         self.msgList = []
         self.metadata = {}
         self.nbMsg = 0
         self.msg = ''
         self.msgName = ''
         self.name = args['name']
         self.ftpConn = FTP()
         if args.has_key('exitOnError'):
             self.exitOnError = args['exitOnError']
         if self.filePath[-1] != '/':
             self.filePath = self.filePath + '/'
         if args['ftpPort'] != '':
             reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' +  "Using provided port for FTP connection : " + args['ftpPort'])
             self.ftpPort = int(args['ftpPort'])
         else:
             reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' +  "No port specified for FTP connection. Using default : 21")
     except KeyError:
         reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Error on ftpSource initialization")
         reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Parameter " + str(sys.exc_info()[1]) + " is missing on source definition. Exiting..." )
         sys.exit(1)
     except :
         reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Unknown error on ftpSource initialization. Exiting...")
         sys.exit(1)        
Esempio n. 10
0
 def process (self):
     try:
         msg = StringIO.StringIO()
         msg.write(self.msg)    
         msg.seek(0)
         self.updateAttributesFromMetadata()
         reticLog.logInfo(self.logList, "( " + self.name + " ) Processing file in sink " + self.name)
         if self.addTimestamp == 'y':
             index = 0
             i = 0
             while index >= 0:
                 i = index
                 index = string.find(self.fileName,'.',index+1)
             now = time.time()
             timestamp = str(time.localtime()[0])+reticUtils.completeWithZero(str(time.localtime()[1]))+reticUtils.completeWithZero(str(time.localtime()[2]))+reticUtils.completeWithZero(str(time.localtime()[3]))+reticUtils.completeWithZero(str(time.localtime()[4]))+reticUtils.completeWithZero(str(time.localtime()[5]))+str(int(random.random()*10000))
             reticLog.logDebug(self.logList, "Timestamp added to filename : " + timestamp)
             new_filename = self.fileName[:i] + timestamp + self.fileName[i:]
         else:
             new_filename = self.fileName
         if self.writeMethod == 'overwrite':
             if self.zip == 'y':                    
                 reticLog.logDebug(self.logList, "Message is being zipped")
                 tmpFpName = '_tmp' + str(int(random.random()*100000000)) + '.dat'
                 tmpFpName2 = '_tmp' + str(int(random.random()*100000000)) + '.dat'
                 tmpFpMsg = open(tmpFpName2,'w+b')
                 tmpFpMsg.write(self.msg)
                 tmpFpMsg.close()
                 tmpZipFile = zipfile.ZipFile(os.path.join(os.getcwd(),tmpFpName),'w',zipfile.ZIP_DEFLATED)
                 tmpZipFile.write(tmpFpName2,self.fileName)
                 
                 tmpZipFile.close()
                 tmpZipFile = open(os.path.join(os.getcwd(),tmpFpName),'r+b')
     
                 self.msg = tmpZipFile.read()
                 tmpZipFile.close()                        
                 os.remove(tmpFpName)
                 os.remove(tmpFpName2)
                 new_filename = new_filename + '.zip'
                 self.fp = open(os.path.join(self.filePath,new_filename),'wb')
         
             elif reticUtils.istext(msg):
                 reticLog.logDebug(self.logList, "Message identified as being text")
                 self.fp = open(os.path.join(self.filePath,new_filename),'w')
             else:
                 reticLog.logDebug(self.logList, "Message identified as being binary data")
                 self.fp = open(os.path.join(self.filePath,new_filename),'wb')
             
             self.fp.truncate()
             self.fp.write(self.msg)
             self.fp.close()
             reticLog.logInfo(self.logList, "( " + self.name + " ) New file created by sink " + self.name + " : " + self.filePath + new_filename)
         else:
             if self.fileType == 'flat':
                 file_content = self.msg
                 self.fp.write(file_content)
             else:
                 msg.seek(0)
                 addLines = msg.readlines()
                 if len(self.lastLine) > 0:
                     self.fp.seek(-(len(self.lastLine)+1),2)
                     self.fp.truncate()
                     del addLines[0:2]
                 self.fp.writelines(addLines)
                 self.fp.flush()
                 self.lastLine = addLines[-1]
             reticLog.logInfo(self.logList, "( " + self.name + " ) Data appended to file in sink " + self.name + " - file : " + self.filePath + new_filename)                                
         return 0
     except:
         errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
         reticLog.logError(self.logList, "( " + self.name + " ) Error during message processing in sink : " + self.name)                        
         reticLog.logError(self.logList, "( " + self.name + " ) " + errorMessage)
         return 1
Esempio n. 11
0
 def start (self):
     'Start the source of the adaptor (begin work...)'
     reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' +  "Starting the source : " + self.name)
     self.getMsg()
     return 0