def __init__(self):
logs_db_name = "LogsAllInOne"
self.conn_logs_all_in_one = Conn.MongodbConn(logs_db_name,
host_name="172.26.11.127")
ip_db = "IPLocationDB"
self.conn_ip_location = Conn.MongodbConn(ip_db,
host_name="172.26.178.208")
self.ip_db_col_name = "IPLocationInfo"
logs_with_ip_db = "LogsWithIPLocation"
self.conn_log_with_ip = Conn.MongodbConn(logs_with_ip_db,
host_name="127.0.0.1")
class MongodbSave():
"""
new connection
"""
def __init__(self):
col_names = DBGlobalSettings.col_names
db_name = DBGlobalSettings.db_name
self.conn = Conn(db_name)
self.db = self.conn.get_db()
"""
save data to mongodb
"""
def save(self):
#get one .tar.gz file
proxy001_tarfiles = LogFilesTarGzGlob.test_glob3()
tar_num = len(proxy001_tarfiles)
for i in range(tar_num):
#get 1 gzfile.
gzfile = proxy001_tarfiles[i]
#gzfile open.
gzfileinfo = GzFileInfo(gzfile)
gzfile = ExtractFiles(gzfileinfo.gzfilename)
print "gzfileinfo: " + gzfileinfo.gzfilename
#log file extract one by one
for extractedfileobj in gzfile.extract_files():
cur_file = gzfile.cur_extract_file
print cur_file
#db collection resetting.
table = self.conn.set_cur_col(cur_file)
log_data = LogDataLineSplit(extractedfileobj)
for data in log_data.read_line():
#build structure
item = LogDataStructure(data).log_data
item.__setitem__("log_server", gzfileinfo.log_server)
item.__setitem__("log_date", gzfileinfo.log_date)
table.insert(item)
class MongodbSave():
"""
new connection
"""
def __init__(self):
col_names = DBGlobalSettings.col_names
db_name = DBGlobalSettings.db_name
self.conn = Conn(db_name)
self.db = self.conn.get_db()
"""
save data to mongodb
"""
def save(self):
#get one .tar.gz file
proxy001_tarfiles = LogFilesTarGzGlob.test_glob3()
tar_num = len(proxy001_tarfiles)
for i in range(tar_num):
#get 1 gzfile.
gzfile = proxy001_tarfiles[i]
#gzfile open.
gzfileinfo = GzFileInfo(gzfile)
gzfile = ExtractFiles(gzfileinfo.gzfilename)
print "gzfileinfo: " + gzfileinfo.gzfilename
#log file extract one by one
for extractedfileobj in gzfile.extract_files():
cur_file = gzfile.cur_extract_file
print cur_file
#db collection resetting.
table = self.conn.set_cur_col(cur_file)
log_data = LogDataLineSplit(extractedfileobj)
for data in log_data.read_line():
#build structure
item = LogDataStructure(data).log_data
item.__setitem__("log_server", gzfileinfo.log_server)
item.__setitem__("log_date", gzfileinfo.log_date)
table.insert(item)
#this script is created for ---- get distinct ip address from all the logs ,
# and then get the location info from ip2location.com ,store them into the mongodb.
from mongobase import Conn
__author__ = 'fandongyun'
from utils import IpLocation
from utils import IpLocationFetch
from mongobase.Settings import DBColNames
from pymongo import mongo_client
from requests import ConnectionError
log_conn = Conn("NginxEventLog")
log_db = log_conn.get_db()
client = mongo_client.MongoClient(host="172.26.178.208")
ip_db = client.__getattr__("IPLocationDB")
ip_db_col_name = "IPLocationInfo"
def find_ip_from_collection(collection_name):
distinct_ip = log_db.__getattr__(collection_name).distinct("remote_addr")
print len(distinct_ip)
return distinct_ip
def is_ip_resolved(ip):
#TODO
ip_count = ip_db.__getattr__(ip_db_col_name).find({"ip": ip}).count()
if ip_count > 0:
#this script is created for ---- get distinct ip address from all the logs ,
# and then get the location info from ip2location.com ,store them into the mongodb.
from mongobase import Conn
__author__ = 'fandongyun'
from utils import IpLocation
from utils import IpLocationFetch
from mongobase.Settings import DBColNames
from pymongo import mongo_client
from requests import ConnectionError
log_conn = Conn("NginxEventLog")
log_db = log_conn.get_db()
client = mongo_client.MongoClient(host= "172.26.178.208")
ip_db = client.__getattr__("IPLocationDB")
ip_db_col_name = "IPLocationInfo"
def find_ip_from_collection(collection_name):
distinct_ip = log_db.__getattr__(collection_name).distinct("remote_addr")
print len(distinct_ip)
return distinct_ip
def is_ip_resolved(ip):
#TODO
ip_count = ip_db.__getattr__(ip_db_col_name).find({"ip": ip}).count()
if ip_count > 0:
def __init__(self):
self.conn = Conn.MongodbConn(db_name=settings().db_name)
self.db = self.conn.get_db()
def __init__(self):
col_names = Settings.DBGlobalSettings.col_names
db_name = "LogsAllInOne"
self.conn = Conn.MongodbConn(db_name)
self.db = self.conn.get_db()
def __init__(self):
col_names = DBGlobalSettings.col_names
db_name = DBGlobalSettings.db_name
self.conn = Conn(db_name)
self.db = self.conn.get_db()
def __init__(self):
col_names = DBGlobalSettings.col_names
db_name = DBGlobalSettings.db_name
self.conn = Conn(db_name)
self.db = self.conn.get_db()
