class User_id_ip_Stat():
def __init__(self):
"""
analysis the IP and User & location
"""
self.setting = DBSetting.DBGlobalSettings()
self.conn_log = MongodbConn(self.setting.db_name,
host_name="172.26.11.127")
self.collection_log = self.conn_log.set_cur_col(self.setting.col_name)
self.conn_uuid_ip = MongodbConn("uuid_ip", host_name="172.26.178.208")
self.collection_uuid_ip = self.conn_uuid_ip.set_cur_col("uuid_ip")
def update_ip_user_count(self):
import time
start = time.time()
log_num = 0
#log_db.__getattr__(collection_name).distinct("remote_addr")
for log_data in self.collection_log.find(
{
"user_agent": {
"$in": ["13DdnAI", "13IprAA"]
}
}, {
"remote_addr": 1,
"http_uuid": 1
}).distinct("http_uuid"):
print log_data
#self.collection_uuid_ip.insert(log_data)
log_num += 1
if log_num % 10000 == 0:
print log_num, "% ", log_num / 32000000., time.time(
) - start, log_num / (time.time() - start)
def __init__(self):
"""
analysis the IP and User & location
"""
self.setting = DBSetting.DBGlobalSettings()
self.conn_log = MongodbConn(self.setting.db_name, host_name="172.26.11.127")
self.collection_log = self.conn_log.set_cur_col(self.setting.col_name)
self.conn_ip = MongodbConn("IPLocationUserCount", host_name="172.26.178.208")
self.collection_ip = self.conn_ip.set_cur_col("IPLocationInfo")
class GlobalStat():
"""
get the global statistics for chart module and geraldo report.
1.total visits
2.visits per month
3.
"""
def __init__(self):
self.setting = DBSetting.DBGlobalSettings()
self.conn = MongodbConn(self.setting.db_name, host_name=self.setting.hostname)
self.collection = self.conn.set_cur_col(self.setting.col_name)
def total_visit_per_month(self, year, month, day):
"""
different from the userStat module.
this function return the HTTP requests that Nginx servers get in One month from start_date.
"""
dates = []
delta_one_day = timedelta(1)
date_begin = datetime.date(year, month, day)
for i in range(30):
date_begin += delta_one_day
dates.append(str(date_begin))
visits = 0
for i in range(len(dates)):
visits += self.collection.find({'log_date': dates[i]}).count()
return visits
def total_visit(self):
return self.collection.find().count()
class ServerStat():
"""
1.visits per server
2.distinct user distribution on each server
3.client type on each server, diff client user likes diff module features.
4.server fail times. http 400 401 500 503 505
5.users stay on time.
6.traffic busy time-line.
7.
"""
def __init__(self):
self.setting = DBSetting.DBGlobalSettings()
self.conn = MongodbConn(self.setting.db_name,
host_name=self.setting.hostname)
self.collection = self.conn.set_cur_col(self.setting.col_name)
self.servers = {
"sns_uuid_token": "sns_uuid_token",
"search_uuid_token": "search_uuid_token",
"gis_uuid_token": "gis_uuid_token",
"dpa_uuid_token": "dpa_uuid_token",
"direction_uuid_token": "direction_uuid_token",
"ctm_uuid_token": "ctm_uuid_token",
"auth_uuid_token": "auth_uuid_token",
"admin_web_uuid_token": "admin_web_uuid_token",
"admin_api_uuid_token": "admin_api_uuid_token",
"access_uuid_token": "access_uuid_token",
"web_uuid_token": "web_uuid_token",
"update_uuid_token": "update_uuid_token",
"traffic_uuid_token": "traffic_uuid_token",
"tile_uuid_token": "tile_uuid_token"
}
def get_log_num_per_server(self, server_name):
return self.collection.find({"col_name": server_name}).count()
def visit_flow_of_servers(self):
# TODO
return NotImplemented
def client_visit_dist_on_server(self, server_name):
"""
client type distribution on specific server_name
"""
# TODO
return NotImplemented
def server_fail_times_per_date(self):
# TODO
return NotImplemented
def traffic_flow_in_one_day(self):
# TODO
return NotImplemented
def traffic_flow_in_month(self):
# TODO
return NotImplemented
class ServerStat():
"""
1.visits per server
2.distinct user distribution on each server
3.client type on each server, diff client user likes diff module features.
4.server fail times. http 400 401 500 503 505
5.users stay on time.
6.traffic busy time-line.
7.
"""
def __init__(self):
self.setting = DBSetting.DBGlobalSettings()
self.conn = MongodbConn(self.setting.db_name, host_name=self.setting.hostname)
self.collection = self.conn.set_cur_col(self.setting.col_name)
self.servers = {"sns_uuid_token": "sns_uuid_token",
"search_uuid_token": "search_uuid_token",
"gis_uuid_token": "gis_uuid_token",
"dpa_uuid_token": "dpa_uuid_token",
"direction_uuid_token": "direction_uuid_token",
"ctm_uuid_token": "ctm_uuid_token",
"auth_uuid_token": "auth_uuid_token",
"admin_web_uuid_token": "admin_web_uuid_token",
"admin_api_uuid_token": "admin_api_uuid_token",
"access_uuid_token": "access_uuid_token",
"web_uuid_token": "web_uuid_token",
"update_uuid_token": "update_uuid_token",
"traffic_uuid_token": "traffic_uuid_token",
"tile_uuid_token": "tile_uuid_token"
}
def get_log_num_per_server(self, server_name):
return self.collection.find({"col_name": server_name}).count()
def visit_flow_of_servers(self):
# TODO
return NotImplemented
def client_visit_dist_on_server(self, server_name):
"""
client type distribution on specific server_name
"""
# TODO
return NotImplemented
def server_fail_times_per_date(self):
# TODO
return NotImplemented
def traffic_flow_in_one_day(self):
# TODO
return NotImplemented
def traffic_flow_in_month(self):
# TODO
return NotImplemented
def __init__(self):
self.setting = DBSetting.DBGlobalSettings()
self.conn = MongodbConn(self.setting.db_name,
host_name=self.setting.hostname)
self.collection = self.conn.set_cur_col(self.setting.col_name)
self.servers = {
"sns_uuid_token": "sns_uuid_token",
"search_uuid_token": "search_uuid_token",
"gis_uuid_token": "gis_uuid_token",
"dpa_uuid_token": "dpa_uuid_token",
"direction_uuid_token": "direction_uuid_token",
"ctm_uuid_token": "ctm_uuid_token",
"auth_uuid_token": "auth_uuid_token",
"admin_web_uuid_token": "admin_web_uuid_token",
"admin_api_uuid_token": "admin_api_uuid_token",
"access_uuid_token": "access_uuid_token",
"web_uuid_token": "web_uuid_token",
"update_uuid_token": "update_uuid_token",
"traffic_uuid_token": "traffic_uuid_token",
"tile_uuid_token": "tile_uuid_token"
}
class ClientStat():
"""
1.client type distribution
2.the ddn and ipro distribution
3.ddn client version distribution
4.ipro client version distribution
"""
def __init__(self):
self.setting = DBSetting.DBGlobalSettings()
self.conn = MongodbConn(self.setting.db_name, host_name=self.setting.hostname)
self.collection = self.conn.set_cur_col(self.setting.col_name)
def client_types(self):
return self.collection.distinct("user_agent")
def client_dist(self):
agent_list = self.client_types()
client_num_dict = {}
for i in agent_list:
n = self.collection.find({"user_agent": i}).count()
client_num_dict.__setitem__(i, n)
return client_num_dict
def client_version(self, client_type):
return self.collection.find({"user_agent": client_type}).distinct("user_agent_ver")
def client_version_dist(self, client_type):
client_type_num = {}
client_versions = self.client_version(client_type)
for i in client_versions:
n = self.collection.find({"user_agent": client_type, "user_agent_ver": i}).count()
client_type_num.__setitem__(i,n)
return client_type_num
def ipro_ddn_dist(self):
client_distribution = self.client_dist()
ipro_ddn_num = {}
ipro_ddn_num.__setitem__("total", sum(client_distribution.values()))
ipro_ddn_num.__setitem__(self.setting.ipro, client_distribution[self.setting.ipro])
ipro_ddn_num.__setitem__(self.setting.ddn, client_distribution[self.setting.ddn])
return ipro_ddn_num
def ipro_ver_dist(self):
return self.client_version_dist(self.setting.ipro)
def ddn_ver_dist(self):
return self.client_version_dist(self.setting.ddn)
class User_id_ip_Stat():
def __init__(self):
"""
analysis the IP and User & location
"""
self.setting = DBSetting.DBGlobalSettings()
self.conn_log = MongodbConn(self.setting.db_name, host_name="172.26.11.127")
self.collection_log = self.conn_log.set_cur_col(self.setting.col_name)
self.conn_uuid_ip = MongodbConn("uuid_ip", host_name="172.26.178.208")
self.collection_uuid_ip = self.conn_uuid_ip.set_cur_col("uuid_ip")
def update_ip_user_count(self):
import time
start = time.time()
log_num = 0
#log_db.__getattr__(collection_name).distinct("remote_addr")
for log_data in self.collection_log.find({"user_agent": {"$in": ["13DdnAI","13IprAA"]}}, {"remote_addr": 1, "http_uuid": 1}).distinct("http_uuid"):
print log_data
#self.collection_uuid_ip.insert(log_data)
log_num += 1
if log_num % 10000 == 0:
print log_num, "% ", log_num/32000000., time.time()-start, log_num/(time.time()-start)
def __init__(self):
self.setting = DBSetting.DBGlobalSettings()
self.conn = MongodbConn(self.setting.db_name, host_name=self.setting.hostname)
self.collection = self.conn.set_cur_col(self.setting.col_name)
self.servers = {"sns_uuid_token": "sns_uuid_token",
"search_uuid_token": "search_uuid_token",
"gis_uuid_token": "gis_uuid_token",
"dpa_uuid_token": "dpa_uuid_token",
"direction_uuid_token": "direction_uuid_token",
"ctm_uuid_token": "ctm_uuid_token",
"auth_uuid_token": "auth_uuid_token",
"admin_web_uuid_token": "admin_web_uuid_token",
"admin_api_uuid_token": "admin_api_uuid_token",
"access_uuid_token": "access_uuid_token",
"web_uuid_token": "web_uuid_token",
"update_uuid_token": "update_uuid_token",
"traffic_uuid_token": "traffic_uuid_token",
"tile_uuid_token": "tile_uuid_token"
}
class UserStat():
def __init__(self):
"""
analysis the IP and User & location
"""
self.setting = DBSetting.DBGlobalSettings()
self.conn_log = MongodbConn(self.setting.db_name, host_name="172.26.11.127")
self.collection_log = self.conn_log.set_cur_col(self.setting.col_name)
self.conn_ip = MongodbConn("IPLocationUserCount", host_name="172.26.178.208")
self.collection_ip = self.conn_ip.set_cur_col("IPLocationInfo")
"""
def update_ip_user_count(self):
import time
start = time.time()
log_num = 0
for log_data in self.collection_log.find().skip(3914916):
self.collection_ip.update({"ip": log_data['remote_addr']}, {"$inc": {"quantity": 1}})
log_num += 1
if log_num % 10000 == 0:
print log_num, "% ", log_num/32000000., time.time()-start, log_num/(time.time()-start)
"""
def geo_global_dist(self):
"""
find the ip distribution based on users' ip access dist.
map - reduce : sum(quantity)
aggregate( { $group :
{ _id : "$state",
totalPop : { $sum : "$pop" } } },
{ $match : {totalPop : { $gte : 10*1000*1000 } } } )
pipe = [
{'$match':{'_Program':DBRef('Programs',p['_id']),'Duration':{'$gt':0}}},
{'$group':{'_id':'$_Program', 'AverageDuration':{'$avg':'$Duration'}}}
]
eps = db.Episodes.aggregate(pipeline=pipe)
print eps['result']
"""
return self.collection_ip.aggregate([
{'$match': {'country': 1, 'log_counts': 1}}, # the data we need .
{'$group': {'_id': '$country', 'log_counts': {'$sum': '$quantity'}}} # the calculation formula.
])
def geo_japan_dist(self):
"""
find out the user distribution in japan.
"""
return self.collection_ip.aggregate([
{'$match': {'country': {'$eq': 'japan'}, 'city': 1, 'log_counts': 1}}, # the data we need .
{'$group': {'_id': '$city', 'log_counts': {'$sum': '$quantity'}}} # the calculation formula.
])
def daily_visit_dist(self):
"""
find out the daily visit traffic
"""
return self.collection_log.aggregate([
{'$match': {'log_date': 1, 'log_counts': 1}}, # the data we need .
{'$group': {'_id': '$log_date', 'log_counts': {'$sum': 1}}} # the calculation formula.
])
def unique_user_visit_dist(self):
"""
find the unique user dist in each day,
"""
return self.collection_log.aggregate([
{'$match': {'http_uuid': 1, 'log_counts': 1}}, # the data we need .
{'$group': {'_id': '$log_date', 'log_counts': {'$sum': '$http_uuid'}}} # the calculation formula.
])
def unique_session_dist(self):
return self.collection_log.aggregate([
{'$match': {'log_date': 1, 'log_counts': 1}}, # the data we need .
{'$group': {'_id': '$http_uuid', 'log_counts': {'$sum': '$http_session'}}} # the calculation formula.
])
def frequency_login_dist(self):
return self.collection_log.aggregate([
{'$match': {'log_date': 1, 'log_counts': 1}}, # the data we need .
{'$group': {'_id': '$http_uuid', 'log_counts': {'$sum': 1}}} # the calculation formula.
])
def __init__(self):
self.setting = DBSetting.DBGlobalSettings()
self.conn = MongodbConn(self.setting.db_name, host_name=self.setting.hostname)
self.collection = self.conn.set_cur_col(self.setting.col_name)