def __init__ (self, args, logger):
try:
self.args = args
self.msg = ''
self.logList = logger
self.name = args['name']
self.dbFactoryArg = args['dbFactroyArg']
self.inputFormat = args['inputFormat']
self.delimiter = args['delimiter']
self.hasHeader = args['hasHeader']
#self.replaceEmptyFieldBy = args['replaceEmptyFieldBy']
if not self.args.has_key('autoCommit'):
self.autocommit = False
#session pool
self.session = None
self.nbThreads = int(args['nbThreads'])
if self.nbThreads == 0:
self.nbThreads = 1
if args.has_key('retries'):
self.retries = int(args['retries'])
else:
self.retries = 5
#self.curArgs = {}
#self.curArgs['cursorName'] = 'cursor1'
self.metadata = {}
self.fieldNames = []
self.fieldLength = []
if self.inputFormat == 'delimited':
if self.args.has_key('fieldNames'):
self.fieldNames = args['fieldNames']
elif self.inputFormat == 'fixedLength':
if self.args.has_key('fieldNames') and self.args.has_key('fieldLength'):
self.fieldNames = args['fieldNames']
self.fieldLength = args['fieldLength']
#self.args['cursorName'] = "cursor"
#args['cursorName'] = "cursor"
#here, for multithread updating database, speed up here
self.connection = dbFactory.dbFactory(args['dbFactroyArg'], self.logList)
#for i in range(self.nbThreads):
self.session = self.makeSession(self.connection)
self.parallelize = 'n'
self.nbQueriesParal = 10
########################################################
if args.has_key('parallelize'):
self.parallelize = args['parallelize']
if args.has_key('nbQueriesParal'):
self.nbQueriesParal = int(args['nbQueriesParal'])
except KeyError:
reticLog.logError(self.logList, "( " + self.name + " ) Error during SQLSink initialization")
reticLog.logError(self.logList, "( " + self.name + " ) Parameter " + str(sys.exc_info()[1]) + " is missing in sink definition" )
sys.exit(1)
except:
errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
reticLog.logError(self.logList, "( " + self.name + " ) Unknown error during initialization in sink")
reticLog.logError(self.logList, "( " + self.name + " ) " + errorMessage)
sys.exit(1)
def __init__ (self, args, logger):
try:
self.logList = logger
self.name = args['name']
self.args = args
reticLog.logInfo(self.logList, '( ' + self.name + ' ) ' + "Intitializing SQLTreeSource : " + self.name)
self.exitOnError = 'y'
self.xpath = XPathAPI()
self.metadata = {}
self.dsn = args['dsn']
self.user = args['user']
self.password = args['password']
self.dbType = args['dbType']
# self.db = args['db']
self.msgSize = args['msgSize']
self.treeQ = args['treeQ']
self.rootTag = args['rootTag']
self.encoding = args['encoding']
self.msgSize = float(args['msgSize'])
self.msgList = []
self.fieldList = []
self.nbMsg = 0
self.tmpFileName = 'tmp_SQLTreeSource' + str(int(random.random()*100000000)) + '.dat'
# self.msg = open(self.tmpFileName,'w+')
self.msg = ''
self.tmpMsg = StringIO.StringIO()
self.msgName = ''
self.curArgs = {}
self.dbConnection = dbFactory.dbFactory(args, self.logList)
self.dbConnection.connect(args)
if args.has_key('exitOnError'):
self.exitOnError = args['exitOnError']
except KeyError:
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Error on SQLTreeSource initialization")
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Parameter " + str(sys.exc_info()[1]) + " is missing on source definition" )
sys.exit(1)
except:
errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + "Unknown error on SQLTreeSource initialization. Exiting...")
reticLog.logError(self.logList, '( ' + self.name + ' ) ' + errorMessage)
sys.exit(1)
def __init__ (self, args, logger):
try:
self.args = args
self.msg = ''
self.logList = logger
self.name = args['name']
self.dbFactoryArg = args['dbFactroyArg']
self.inputFormat = args['inputFormat']
self.delimiter = args['delimiter']
self.hasHeader = args['hasHeader']
#self.replaceEmptyFieldBy = args['replaceEmptyFieldBy']
if not self.args.has_key('autoCommit'):
self.autocommit = False
#session pool
self.session = None
self.nbThreads = int(args['nbThreads'])
if self.nbThreads == 0:
self.nbThreads = 1
if args.has_key('retries'):
self.retries = int(args['retries'])
else:
self.retries = 5
#self.curArgs = {}
#self.curArgs['cursorName'] = 'cursor1'
self.metadata = {}
self.fieldNames = []
self.fieldLength = []
if self.inputFormat == 'delimited':
if self.args.has_key('fieldNames'):
self.fieldNames = args['fieldNames']
elif self.inputFormat == 'fixedLength':
if self.args.has_key('fieldNames') and self.args.has_key('fieldLength'):
self.fieldNames = args['fieldNames']
self.fieldLength = args['fieldLength']
self.connection = dbFactory.dbFactory(args['dbFactroyArg'], self.logList)
# map DataValue table in ODM database to DataValue python class
# to utilize the magic power of sqlalchemy
metadata = schema.MetaData()
metadata.bind = self.connection.engine
DataValues_table = schema.Table('DataValues',metadata, \
autoload=True,autoload_with=self.connection.engine)
orm.mapper(DataValues,DataValues_table)
Sites_table = schema.Table('Sites',metadata, \
autoload=True,autoload_with=self.connection.engine)
orm.mapper(Sites,Sites_table)
Variables_table = schema.Table('Variables',metadata, \
autoload=True,autoload_with=self.connection.engine)
orm.mapper(Variables,Variables_table)
Methods_table = schema.Table('Methods',metadata, \
autoload=True,autoload_with=self.connection.engine)
orm.mapper(Methods,Methods_table)
#for i in range(self.nbThreads):
self.session = self.makeSession(self.connection)
self.parallelize = 'n'
self.nbQueriesParal = 10
########################################################
if args.has_key('parallelize'):
self.parallelize = args['parallelize']
if args.has_key('nbQueriesParal'):
self.nbQueriesParal = int(args['nbQueriesParal'])
except KeyError:
reticLog.logError(self.logList, "( " + self.name + " ) Error during SQLSink initialization")
reticLog.logError(self.logList, "( " + self.name + " ) Parameter " + str(sys.exc_info()[1]) + " is missing in sink definition" )
sys.exit(1)
except:
errorMessage = traceback.format_exception_only(sys.exc_info()[0],sys.exc_info()[1])[0]
reticLog.logError(self.logList, "( " + self.name + " ) Unknown error during initialization in sink")
reticLog.logError(self.logList, "( " + self.name + " ) " + errorMessage)
sys.exit(1)
