Exemplo n.º 1
0
    def __init__(self):

        logs_db_name = "LogsAllInOne"
        self.conn_logs_all_in_one = Conn.MongodbConn(logs_db_name,
                                                     host_name="172.26.11.127")

        ip_db = "IPLocationDB"
        self.conn_ip_location = Conn.MongodbConn(ip_db,
                                                 host_name="172.26.178.208")
        self.ip_db_col_name = "IPLocationInfo"

        logs_with_ip_db = "LogsWithIPLocation"
        self.conn_log_with_ip = Conn.MongodbConn(logs_with_ip_db,
                                                 host_name="127.0.0.1")
Exemplo n.º 2
0
class MongodbSave():
    """
    new connection
    """
    def __init__(self):
        col_names = DBGlobalSettings.col_names
        db_name = DBGlobalSettings.db_name

        self.conn = Conn(db_name)
        self.db = self.conn.get_db()

    """
        save data to mongodb

    """

    def save(self):
        #get one .tar.gz file

        proxy001_tarfiles = LogFilesTarGzGlob.test_glob3()
        tar_num = len(proxy001_tarfiles)

        for i in range(tar_num):
            #get 1 gzfile.
            gzfile = proxy001_tarfiles[i]

            #gzfile open.
            gzfileinfo = GzFileInfo(gzfile)
            gzfile = ExtractFiles(gzfileinfo.gzfilename)

            print "gzfileinfo: " + gzfileinfo.gzfilename

            #log file extract one by one
            for extractedfileobj in gzfile.extract_files():

                cur_file = gzfile.cur_extract_file

                print cur_file
                #db collection resetting.
                table = self.conn.set_cur_col(cur_file)

                log_data = LogDataLineSplit(extractedfileobj)

                for data in log_data.read_line():
                    #build structure
                    item = LogDataStructure(data).log_data
                    item.__setitem__("log_server", gzfileinfo.log_server)
                    item.__setitem__("log_date", gzfileinfo.log_date)
                    table.insert(item)
Exemplo n.º 3
0
class MongodbSave():
    """
    new connection
    """
    def __init__(self):
        col_names = DBGlobalSettings.col_names
        db_name = DBGlobalSettings.db_name

        self.conn = Conn(db_name)
        self.db = self.conn.get_db()

    """
        save data to mongodb

    """
    def save(self):
        #get one .tar.gz file

        proxy001_tarfiles = LogFilesTarGzGlob.test_glob3()
        tar_num = len(proxy001_tarfiles)

        for i in range(tar_num):
            #get 1 gzfile.
            gzfile = proxy001_tarfiles[i]

            #gzfile open.
            gzfileinfo = GzFileInfo(gzfile)
            gzfile = ExtractFiles(gzfileinfo.gzfilename)

            print "gzfileinfo: " + gzfileinfo.gzfilename

            #log file extract one by one
            for extractedfileobj in gzfile.extract_files():

                cur_file = gzfile.cur_extract_file

                print cur_file
                #db collection resetting.
                table = self.conn.set_cur_col(cur_file)

                log_data = LogDataLineSplit(extractedfileobj)

                for data in log_data.read_line():
                    #build structure
                    item = LogDataStructure(data).log_data
                    item.__setitem__("log_server", gzfileinfo.log_server)
                    item.__setitem__("log_date", gzfileinfo.log_date)
                    table.insert(item)
#this script is created for ---- get distinct ip address from all the logs ,
# and then get the location info from ip2location.com ,store them into the mongodb.
from mongobase import Conn

__author__ = 'fandongyun'

from utils import IpLocation
from utils import IpLocationFetch

from mongobase.Settings import DBColNames
from pymongo import mongo_client
from requests import ConnectionError

log_conn = Conn("NginxEventLog")
log_db = log_conn.get_db()

client = mongo_client.MongoClient(host="172.26.178.208")
ip_db = client.__getattr__("IPLocationDB")
ip_db_col_name = "IPLocationInfo"


def find_ip_from_collection(collection_name):
    distinct_ip = log_db.__getattr__(collection_name).distinct("remote_addr")
    print len(distinct_ip)
    return distinct_ip


def is_ip_resolved(ip):
    #TODO
    ip_count = ip_db.__getattr__(ip_db_col_name).find({"ip": ip}).count()
    if ip_count > 0:
#this script is created for ---- get distinct ip address from all the logs ,
# and then get the location info from ip2location.com ,store them into the mongodb.
from mongobase import Conn

__author__ = 'fandongyun'

from utils import IpLocation
from utils import IpLocationFetch

from mongobase.Settings import DBColNames
from pymongo import mongo_client
from requests import ConnectionError


log_conn = Conn("NginxEventLog")
log_db = log_conn.get_db()

client = mongo_client.MongoClient(host= "172.26.178.208")
ip_db = client.__getattr__("IPLocationDB")
ip_db_col_name = "IPLocationInfo"


def find_ip_from_collection(collection_name):
    distinct_ip = log_db.__getattr__(collection_name).distinct("remote_addr")
    print len(distinct_ip)
    return distinct_ip

def is_ip_resolved(ip):
    #TODO
    ip_count = ip_db.__getattr__(ip_db_col_name).find({"ip": ip}).count()
    if ip_count > 0:
Exemplo n.º 6
0
 def __init__(self):
     self.conn = Conn.MongodbConn(db_name=settings().db_name)
     self.db = self.conn.get_db()
Exemplo n.º 7
0
    def __init__(self):
        col_names = Settings.DBGlobalSettings.col_names
        db_name = "LogsAllInOne"

        self.conn = Conn.MongodbConn(db_name)
        self.db = self.conn.get_db()
Exemplo n.º 8
0
    def __init__(self):
        col_names = DBGlobalSettings.col_names
        db_name = DBGlobalSettings.db_name

        self.conn = Conn(db_name)
        self.db = self.conn.get_db()
Exemplo n.º 9
0
    def __init__(self):
        col_names = DBGlobalSettings.col_names
        db_name = DBGlobalSettings.db_name

        self.conn = Conn(db_name)
        self.db = self.conn.get_db()