def __init__(self): logs_db_name = "LogsAllInOne" self.conn_logs_all_in_one = Conn.MongodbConn(logs_db_name, host_name="172.26.11.127") ip_db = "IPLocationDB" self.conn_ip_location = Conn.MongodbConn(ip_db, host_name="172.26.178.208") self.ip_db_col_name = "IPLocationInfo" logs_with_ip_db = "LogsWithIPLocation" self.conn_log_with_ip = Conn.MongodbConn(logs_with_ip_db, host_name="127.0.0.1")
class MongodbSave(): """ new connection """ def __init__(self): col_names = DBGlobalSettings.col_names db_name = DBGlobalSettings.db_name self.conn = Conn(db_name) self.db = self.conn.get_db() """ save data to mongodb """ def save(self): #get one .tar.gz file proxy001_tarfiles = LogFilesTarGzGlob.test_glob3() tar_num = len(proxy001_tarfiles) for i in range(tar_num): #get 1 gzfile. gzfile = proxy001_tarfiles[i] #gzfile open. gzfileinfo = GzFileInfo(gzfile) gzfile = ExtractFiles(gzfileinfo.gzfilename) print "gzfileinfo: " + gzfileinfo.gzfilename #log file extract one by one for extractedfileobj in gzfile.extract_files(): cur_file = gzfile.cur_extract_file print cur_file #db collection resetting. table = self.conn.set_cur_col(cur_file) log_data = LogDataLineSplit(extractedfileobj) for data in log_data.read_line(): #build structure item = LogDataStructure(data).log_data item.__setitem__("log_server", gzfileinfo.log_server) item.__setitem__("log_date", gzfileinfo.log_date) table.insert(item)
#this script is created for ---- get distinct ip address from all the logs , # and then get the location info from ip2location.com ,store them into the mongodb. from mongobase import Conn __author__ = 'fandongyun' from utils import IpLocation from utils import IpLocationFetch from mongobase.Settings import DBColNames from pymongo import mongo_client from requests import ConnectionError log_conn = Conn("NginxEventLog") log_db = log_conn.get_db() client = mongo_client.MongoClient(host="172.26.178.208") ip_db = client.__getattr__("IPLocationDB") ip_db_col_name = "IPLocationInfo" def find_ip_from_collection(collection_name): distinct_ip = log_db.__getattr__(collection_name).distinct("remote_addr") print len(distinct_ip) return distinct_ip def is_ip_resolved(ip): #TODO ip_count = ip_db.__getattr__(ip_db_col_name).find({"ip": ip}).count() if ip_count > 0:
#this script is created for ---- get distinct ip address from all the logs , # and then get the location info from ip2location.com ,store them into the mongodb. from mongobase import Conn __author__ = 'fandongyun' from utils import IpLocation from utils import IpLocationFetch from mongobase.Settings import DBColNames from pymongo import mongo_client from requests import ConnectionError log_conn = Conn("NginxEventLog") log_db = log_conn.get_db() client = mongo_client.MongoClient(host= "172.26.178.208") ip_db = client.__getattr__("IPLocationDB") ip_db_col_name = "IPLocationInfo" def find_ip_from_collection(collection_name): distinct_ip = log_db.__getattr__(collection_name).distinct("remote_addr") print len(distinct_ip) return distinct_ip def is_ip_resolved(ip): #TODO ip_count = ip_db.__getattr__(ip_db_col_name).find({"ip": ip}).count() if ip_count > 0:
def __init__(self): self.conn = Conn.MongodbConn(db_name=settings().db_name) self.db = self.conn.get_db()
def __init__(self): col_names = Settings.DBGlobalSettings.col_names db_name = "LogsAllInOne" self.conn = Conn.MongodbConn(db_name) self.db = self.conn.get_db()
def __init__(self): col_names = DBGlobalSettings.col_names db_name = DBGlobalSettings.db_name self.conn = Conn(db_name) self.db = self.conn.get_db()