def process():
## following needs to be simplified.##
try:
except sqlite3.Error as err:
sqlite_logger.error("conntect sqlite database failed, ret = %s" % err.args[0])
##2) cursor
sqlite_cursor = sqlite_conn.cursor()
# sql_desc2 = "DROP TABLE IF EXISTS ?;"
sql_desc2 = """select count(*) from sqlite_master where type= 'table' and name = 'mktinfo';""";
row = sqlite_cursor.fetchone()
if (row[0] is None or row[0] > 0):
# sqlite_cursor.execute(sql_desc2)
# print 'drop table failed'
# sqlite_logger.error("drop table failed, ret = %s" % e.args[0])
# sqlite_cursor.close()
# return
##4) create table
## //**
try:
sqlite_cursor.execute(sql_desc)
sqlite_logger.error("create table failed, ret = %s" % e.args[0])
return
sqlite_conn.commit()
sqlite_logger.debug("create table( mktinfo ) succ.")
##6) clean up datestr
sqlite_cursor.close()
sqlite_conn.close()
scanErrFile.write("Start to scan records on %s...\n" % datestr)
##open files
sql_desc3 = scanCtp.sql_add_record_2
for i in datafile:
re = scanCtp.p.match(line)
if (re == None):
continue
##data constraints
tmp_list = scanCtp.record_2(re)
##insert records
except sqlite3.Error as e:
print 'insert record failed. %s' % e.args[0]
scanErrFile.write(line)
try:
ifile.read()
ifile.close()
def insertRecord(datestr, sqlite_conn, sqlite_cursor):
sqlite_logger.debug("Start to scan records on %s..." % datestr)
scanErrFile.write("Start to scan records on %s...\n" % datestr)
datafile = scanCtp.findTestlogfiles(ABSOLUTE_DIR + datestr)
## open files
sql_desc3 = scanCtp.sql_add_record_2
for i in datafile:
tmp_name = ABSOLUTE_DIR + datestr + "/" + i
ifile = open(tmp_name, "r")
print tmp_name
scanErrFile.write("Begin scanning %s \n" % tmp_name)
sqlite_logger.info("Begin scanning %s " % tmp_name)
datalines = ifile.readlines()
for line in datalines:
# line = "18246655 [139707331954432] DEBUG test <> - 20140627 au1408 263.25 264.05 263.8 212 262.45 263.25 262.45 16 4.2072e+06 220 1.79769e+308 1.79769e+308 277.25 250.8 02:00:15 0 262.75 2 263.3 1 262950 255957"
re = scanCtp.p.match(line)
if re == None:
scanErrFile.write(line)
continue
re = re.groups()
## data constraints
tmp_list = scanCtp.record_2(re)
## insert records
try:
sqlite_cursor.execute(sql_desc3, tmp_list)
except sqlite3.Error as e:
print "insert record failed. %s" % e.args[0]
sqlite_logger.error("Failed insert,ret=%s" % e.args[0])
scanErrFile.write(line)
continue
## clean up datafile
sqlite_conn.commit()
try:
ifile.read()
finally:
ifile.close()
sqlite_logger.info("Finish scanning %s" % i)