def process(self):
'Get the next message to be processed or return that sources are dry'
self.msg = ''
self.tmpMsg = StringIO.StringIO()
try:
parser = DOMParser( )
parser.parse(InputSource(java.io.StringReader(self.InMsg)))
XMLStream = parser.getDocument()
print "message parsed ok"
resultNodeList = self.xpath.selectNodeList(XMLStream,'/*/' + self.treeQ['request']['recTag'])
for i in range(resultNodeList.getLength()):
currentTag = resultNodeList.item(i)
print "query 1"
fieldList = self.xpath.selectNodeList(currentTag,'*')
query = self.treeQ['request']['SQL']
(generatedId, modQuery) = self.prepareInsert(query, fieldList)
print "Generated query : " + str(modQuery)
self.executeQuery(modQuery)
for nextQuery in self.treeQ['request']['nextRequests']:
print "next query"
self.processQuery(nextQuery, generatedId, currentTag)
return 0
except:
errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
reticLog.logError(self.logList, '(' + self.name + ') ' + "Error during message processing" )
reticLog.logError(self.logList, '(' + self.name + ') ' + errorMessage)
return 1
def next(self):
if self.nbMsg == 0:
return 0
else:
try:
reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' + "Starting processing phase of source " + self.name)
self.updateAttributesFromMetadata()
print "updateMetTermine"
SOAPMessage = self.SOAPMessage
for key in self.params.keys():
print "key : " + key
SOAPMessage = string.replace(SOAPMessage,'**' + key[1:] + '**',self.params[key])
print "SOAPMsg : " + SOAPMessage
reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' + "Executing SOAP request")
reticLog.logDebug(self.logList, '( ' + self.name + ' ) ' + "SOAP Request : " + SOAPMessage)
if self.proxyUrl <> '':
urlopener = urllib.FancyURLopener( {'http' : self.proxyUrl } )
else:
urlopener = urllib.URLopener()
urlopener.addheaders.append(('Content-Length', str(len(SOAPMessage))))
urlopener.addheaders.append(('Content-type', 'text/xml; charset="utf-8"'))
urlopener.addheaders.append(('SOAPAction', self.SOAPAction))
resp = urlopener.open(self.url, SOAPMessage)
response = resp.read()
reticLog.logDebug(self.logList, '( ' + self.name + ' ) ' + "SOAP Response : " + response)
self.msg = response
reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' + "source " + self.name + " executed ok")
return 1
except:
errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Error during process phase of source : " + self.name)
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + errorMessage)
return 0
def __init__ (self, args, logger):
try:
self.args = args
self.logList = logger
self.name = args['name']
self.SOAPMessage = args['SOAPMessage']
self.SOAPAction = args['SOAPAction']
self.url = args['url']
self.proxyUrl = args['proxyUrl']
self.params = {}
for key in self.args.keys():
if key[0] == '_':
self.params[key] = self.args[key]
self.exitOnError = args['exitOnError']
self.nbMsg = 0
self.msg = ''
self.msgName = ''
self.metadata = {}
reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' + "Source " + self.name + " initialized.")
except KeyError:
try:
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Error during init phase of source : " + self.name)
except:
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Error during init phase of source")
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Parameter " + str(sys.exc_info()[1]) + " is missing in component definition" )
sys.exit(1)
except:
errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Unknown error during initialization in source : " + self.name)
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + errorMessage)
sys.exit(1)
def getMsg (self, message):
try:
reticLog.logInfo(self.logList, "( " + self.name + " ) Retrieving message for sink : " + self.name)
# Re-initialize msg to get new message
self.msg = ''
self.msg = message
return 0
except:
errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
reticLog.logError(self.logList, "( " + self.name + " ) Error during message retrieval in sink : " + self.name)
reticLog.logError(self.logList, "( " + self.name + " ) " + errorMessage)
return 1
def getMsg (self, message):
'Initializes input buffer with message content'
try:
reticLog.logInfo(self.logList, '(' + self.name + ') ' + "Getting message into sink")
self.InMsg = ''
self.msg = ''
self.InMsg = message
return 0
except:
errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
reticLog.logError(self.logList, '(' + self.name + ') ' + "Error during message retrieval in sink" )
reticLog.logError(self.logList, '(' + self.name + ') ' + errorMessage)
return 1
def start (self):
'Start the source of the adaptor (begin work...)'
reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' + "Starting the source adaptor")
try:
self.connect()
self.getMsg()
self.ftpConn.close()
return 0
except:
errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Error on adaptor start on source : " + self.name)
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + errorMessage)
if self.exitOnError.lower() == 'y':
sys.exit(1)
def commit(self):
'Commit the current message treatment'
if self.nbMsg > 0 :
reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' + "Commiting msg " + self.msgList[0] + " on source : " + self.msgName)
msgName = self.msgList[0]
try:
if self.newExtension != '':
reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' + "Processing file : " + self.msgList[0])
self.connect()
self.ftpConn.sendcmd('RNFR ' + self.msgName)
self.ftpConn.sendcmd('RNTO ' + self.msgName + '.' + self.newExtension)
self.nbMsg = self.nbMsg - 1
self.msgList = self.msgList[1:]
self.msg = ''
return 1
except:
errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Error on commit phase on source - File : " + msgName)
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + errorMessage)
if self.exitOnError.lower() == 'y':
sys.exit(1)
return 0
else:
return 0
def __init__ (self, args, logger):
try:
self.args = args
self.logList = logger
self.name = args['name']
self.msg = ''
self.filePath = args['filePath']
self.fileName = args['fileName']
self.addTimestamp = args['addTimestamp']
self.fileType = 'flat'
self.writeMethod = 'overwrite'
if args.has_key('zip'):
self.zip = args['zip']
else:
self.zip = 'n'
self.args['zip'] = 'n'
self.maxBytes = 0
self.metadata = {}
if args.has_key('fileType'):
self.fileType = args['fileType']
if args.has_key('writeMethod'):
self.writeMethod = args['writeMethod']
if self.fileType.lower() == 'xml' and self.writeMethod.lower() == 'append':
try:
self.fp = open(os.path.join(self.filePath, self.fileName), 'r')
self.lastLine = self.fp.readlines()[-1]
print "Last line : " + self.lastLine
self.fp.close()
self.fp = open(os.path.join(self.filePath,self.fileName), 'a+')
except:
self.lastLine = ''
print "Last line retrieval => error..."
self.fp = open(os.path.join(self.filePath,self.fileName), 'a+')
elif self.fileType.lower() == 'flat' and self.writeMethod.lower() == 'append':
self.fp = open(os.path.join(self.filePath,self.fileName), 'a')
except KeyError:
reticLog.logError(self.logList, "Error during FileSink initialization")
reticLog.logError(self.logList, "Parameter " + str(sys.exc_info()[1]) + " is missing in sink definition" )
sys.exit(1)
except:
errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
reticLog.logError(self.logList, "( " + self.name + " ) Unknown error during initialization in sink")
reticLog.logError(self.logList, "( " + self.name + " ) " + errorMessage)
sys.exit(1)
def next(self):
'Get the next message to be processed or return that sources are dry'
try:
self.connect()
except:
errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Error on message retrieval on source : " + self.name)
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + errorMessage)
if self.exitOnError.lower() == 'y':
sys.exit(1)
if self.nbMsg == 0:
reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' + "Sources dry on source ")
return 0
else:
try:
reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' + "Messages Left on queue of adaptor : " + str(self.nbMsg))
reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' + "Processing file : " + self.msgList[0])
self.msgName = self.msgList[0]
dotIndex = string.find(self.msgName,'.')
if dotIndex > 0:
self.metadata['filename'] = string.split(self.msgName,'.')[0]
self.metadata['extension'] = string.split(self.msgName,'.')[1]
else:
self.metadata['filename'] = self.msgName
self.metadata['extension'] = ''
msg = StringIO.StringIO()
sys.stdout = msg
self.ftpConn.retrlines('RETR ' + self.msgName)
sys.stdout = sys.__stdout__
msg.seek(0)
self.msg = msg.read()
self.ftpConn.close()
return 1
except:
errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Error on message retrieval on source : " + self.name)
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + errorMessage)
if self.exitOnError.lower() == 'y':
sys.exit(1)
return 1
def __init__ (self, args, logger):
try:
self.logList = logger
self.name = args['name']
self.args = args
reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' + "Intitializing SQLTreeSource : " + self.name)
self.exitOnError = 'y'
self.xpath = XPathAPI()
self.metadata = {}
self.dsn = args['dsn']
self.user = args['user']
self.password = args['password']
self.dbType = args['dbType']
# self.db = args['db']
self.msgSize = args['msgSize']
self.treeQ = args['treeQ']
self.rootTag = args['rootTag']
self.encoding = args['encoding']
self.msgSize = float(args['msgSize'])
self.msgList = []
self.fieldList = []
self.nbMsg = 0
self.tmpFileName = 'tmp_SQLTreeSource' + str(int(random.random()*100000000)) + '.dat'
# self.msg = open(self.tmpFileName,'w+')
self.msg = ''
self.tmpMsg = StringIO.StringIO()
self.msgName = ''
self.curArgs = {}
self.dbConnection = dbFactory.dbFactory(args, self.logList)
self.dbConnection.connect(args)
if args.has_key('exitOnError'):
self.exitOnError = args['exitOnError']
except KeyError:
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Error on SQLTreeSource initialization")
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Parameter " + str(sys.exc_info()[1]) + " is missing on source definition" )
sys.exit(1)
except:
errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Unknown error on SQLTreeSource initialization. Exiting...")
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + errorMessage)
sys.exit(1)
def __init__ (self, args, logger):
try:
self.logList = logger
self.name = args['name']
self.exitOnError = 'y'
reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' + "Intitializing ftpSource : " + self.name)
self.ftpHost = args['ftpHost']
self.ftpPort = int(21)
self.ftpUser = args['ftpUser']
self.ftpPass = args['ftpPass']
self.filePath = args['filePath']
self.fileFilter = args['fileFilter']
self.newExtension = args['newExtension']
self.msgList = []
self.metadata = {}
self.nbMsg = 0
self.msg = ''
self.msgName = ''
self.name = args['name']
self.ftpConn = FTP()
if args.has_key('exitOnError'):
self.exitOnError = args['exitOnError']
if self.filePath[-1] != '/':
self.filePath = self.filePath + '/'
if args['ftpPort'] != '':
reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' + "Using provided port for FTP connection : " + args['ftpPort'])
self.ftpPort = int(args['ftpPort'])
else:
reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' + "No port specified for FTP connection. Using default : 21")
except KeyError:
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Error on ftpSource initialization")
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Parameter " + str(sys.exc_info()[1]) + " is missing on source definition. Exiting..." )
sys.exit(1)
except :
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Unknown error on ftpSource initialization. Exiting...")
sys.exit(1)
def process (self):
try:
msg = StringIO.StringIO()
msg.write(self.msg)
msg.seek(0)
self.updateAttributesFromMetadata()
reticLog.logInfo(self.logList, "( " + self.name + " ) Processing file in sink " + self.name)
if self.addTimestamp == 'y':
index = 0
i = 0
while index >= 0:
i = index
index = string.find(self.fileName,'.',index+1)
now = time.time()
timestamp = str(time.localtime()[0])+reticUtils.completeWithZero(str(time.localtime()[1]))+reticUtils.completeWithZero(str(time.localtime()[2]))+reticUtils.completeWithZero(str(time.localtime()[3]))+reticUtils.completeWithZero(str(time.localtime()[4]))+reticUtils.completeWithZero(str(time.localtime()[5]))+str(int(random.random()*10000))
reticLog.logDebug(self.logList, "Timestamp added to filename : " + timestamp)
new_filename = self.fileName[:i] + timestamp + self.fileName[i:]
else:
new_filename = self.fileName
if self.writeMethod == 'overwrite':
if self.zip == 'y':
reticLog.logDebug(self.logList, "Message is being zipped")
tmpFpName = '_tmp' + str(int(random.random()*100000000)) + '.dat'
tmpFpName2 = '_tmp' + str(int(random.random()*100000000)) + '.dat'
tmpFpMsg = open(tmpFpName2,'w+b')
tmpFpMsg.write(self.msg)
tmpFpMsg.close()
tmpZipFile = zipfile.ZipFile(os.path.join(os.getcwd(),tmpFpName),'w',zipfile.ZIP_DEFLATED)
tmpZipFile.write(tmpFpName2,self.fileName)
tmpZipFile.close()
tmpZipFile = open(os.path.join(os.getcwd(),tmpFpName),'r+b')
self.msg = tmpZipFile.read()
tmpZipFile.close()
os.remove(tmpFpName)
os.remove(tmpFpName2)
new_filename = new_filename + '.zip'
self.fp = open(os.path.join(self.filePath,new_filename),'wb')
elif reticUtils.istext(msg):
reticLog.logDebug(self.logList, "Message identified as being text")
self.fp = open(os.path.join(self.filePath,new_filename),'w')
else:
reticLog.logDebug(self.logList, "Message identified as being binary data")
self.fp = open(os.path.join(self.filePath,new_filename),'wb')
self.fp.truncate()
self.fp.write(self.msg)
self.fp.close()
reticLog.logInfo(self.logList, "( " + self.name + " ) New file created by sink " + self.name + " : " + self.filePath + new_filename)
else:
if self.fileType == 'flat':
file_content = self.msg
self.fp.write(file_content)
else:
msg.seek(0)
addLines = msg.readlines()
if len(self.lastLine) > 0:
self.fp.seek(-(len(self.lastLine)+1),2)
self.fp.truncate()
del addLines[0:2]
self.fp.writelines(addLines)
self.fp.flush()
self.lastLine = addLines[-1]
reticLog.logInfo(self.logList, "( " + self.name + " ) Data appended to file in sink " + self.name + " - file : " + self.filePath + new_filename)
return 0
except:
errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
reticLog.logError(self.logList, "( " + self.name + " ) Error during message processing in sink : " + self.name)
reticLog.logError(self.logList, "( " + self.name + " ) " + errorMessage)
return 1