示例#1
0
class MongodbSave():
    """
    new connection
    """
    def __init__(self):
        col_names = DBGlobalSettings.col_names
        db_name = DBGlobalSettings.db_name

        self.conn = Conn(db_name)
        self.db = self.conn.get_db()

    """
        save data to mongodb

    """

    def save(self):
        #get one .tar.gz file

        proxy001_tarfiles = LogFilesTarGzGlob.test_glob3()
        tar_num = len(proxy001_tarfiles)

        for i in range(tar_num):
            #get 1 gzfile.
            gzfile = proxy001_tarfiles[i]

            #gzfile open.
            gzfileinfo = GzFileInfo(gzfile)
            gzfile = ExtractFiles(gzfileinfo.gzfilename)

            print "gzfileinfo: " + gzfileinfo.gzfilename

            #log file extract one by one
            for extractedfileobj in gzfile.extract_files():

                cur_file = gzfile.cur_extract_file

                print cur_file
                #db collection resetting.
                table = self.conn.set_cur_col(cur_file)

                log_data = LogDataLineSplit(extractedfileobj)

                for data in log_data.read_line():
                    #build structure
                    item = LogDataStructure(data).log_data
                    item.__setitem__("log_server", gzfileinfo.log_server)
                    item.__setitem__("log_date", gzfileinfo.log_date)
                    table.insert(item)
class MongodbSave():
    """
    new connection
    """
    def __init__(self):
        col_names = DBGlobalSettings.col_names
        db_name = DBGlobalSettings.db_name

        self.conn = Conn(db_name)
        self.db = self.conn.get_db()

    """
        save data to mongodb

    """
    def save(self):
        #get one .tar.gz file

        proxy001_tarfiles = LogFilesTarGzGlob.test_glob3()
        tar_num = len(proxy001_tarfiles)

        for i in range(tar_num):
            #get 1 gzfile.
            gzfile = proxy001_tarfiles[i]

            #gzfile open.
            gzfileinfo = GzFileInfo(gzfile)
            gzfile = ExtractFiles(gzfileinfo.gzfilename)

            print "gzfileinfo: " + gzfileinfo.gzfilename

            #log file extract one by one
            for extractedfileobj in gzfile.extract_files():

                cur_file = gzfile.cur_extract_file

                print cur_file
                #db collection resetting.
                table = self.conn.set_cur_col(cur_file)

                log_data = LogDataLineSplit(extractedfileobj)

                for data in log_data.read_line():
                    #build structure
                    item = LogDataStructure(data).log_data
                    item.__setitem__("log_server", gzfileinfo.log_server)
                    item.__setitem__("log_date", gzfileinfo.log_date)
                    table.insert(item)