Example #1
0
def process():    
## following needs to be simplified.##
    try:
    except sqlite3.Error as err:
        sqlite_logger.error("conntect sqlite database failed, ret = %s" % err.args[0])
##2) cursor        
    sqlite_cursor = sqlite_conn.cursor()
#    sql_desc2 = "DROP TABLE IF EXISTS ?;"
    sql_desc2 = """select count(*) from sqlite_master where type= 'table' and name = 'mktinfo';""";
    row = sqlite_cursor.fetchone()
    if (row[0] is None or row[0] > 0):
       #      sqlite_cursor.execute(sql_desc2)
       #      print 'drop table failed'
       #      sqlite_logger.error("drop table failed, ret = %s" % e.args[0])
       #      sqlite_cursor.close()
       #      return
##4) create table
## //**
    try:
        sqlite_cursor.execute(sql_desc)        
        sqlite_logger.error("create table failed, ret = %s" % e.args[0])
        return
    sqlite_conn.commit()
    sqlite_logger.debug("create table( mktinfo ) succ.")
##6) clean up datestr
    sqlite_cursor.close()
    sqlite_conn.close()
    scanErrFile.write("Start to scan records on %s...\n" % datestr)
##open files
    sql_desc3 = scanCtp.sql_add_record_2
    for i in datafile:
            re = scanCtp.p.match(line)
            if (re == None):
                continue
##data constraints
            tmp_list = scanCtp.record_2(re)
##insert records
            except sqlite3.Error as e:
                print 'insert record failed. %s' % e.args[0]
                scanErrFile.write(line)
        try:
            ifile.read()
            ifile.close()
Example #2
0
def insertRecord(datestr, sqlite_conn, sqlite_cursor):
    sqlite_logger.debug("Start to scan records on %s..." % datestr)
    scanErrFile.write("Start to scan records on %s...\n" % datestr)
    datafile = scanCtp.findTestlogfiles(ABSOLUTE_DIR + datestr)
    ## open files
    sql_desc3 = scanCtp.sql_add_record_2
    for i in datafile:
        tmp_name = ABSOLUTE_DIR + datestr + "/" + i
        ifile = open(tmp_name, "r")
        print tmp_name
        scanErrFile.write("Begin scanning %s \n" % tmp_name)
        sqlite_logger.info("Begin scanning %s " % tmp_name)
        datalines = ifile.readlines()
        for line in datalines:
            # line = "18246655 [139707331954432] DEBUG test <> - 20140627 au1408   263.25 264.05 263.8 212 262.45 263.25 262.45 16 4.2072e+06 220 1.79769e+308 1.79769e+308 277.25 250.8 02:00:15 0 262.75 2 263.3 1 262950 255957"
            re = scanCtp.p.match(line)
            if re == None:
                scanErrFile.write(line)
                continue
            re = re.groups()
            ## data constraints
            tmp_list = scanCtp.record_2(re)
            ## insert records
            try:
                sqlite_cursor.execute(sql_desc3, tmp_list)
            except sqlite3.Error as e:
                print "insert record failed. %s" % e.args[0]
                sqlite_logger.error("Failed insert,ret=%s" % e.args[0])
                scanErrFile.write(line)
                continue
        ## clean up datafile
        sqlite_conn.commit()
        try:
            ifile.read()
        finally:
            ifile.close()
        sqlite_logger.info("Finish scanning %s" % i)