def get_stats(self):
host = self.config.get('hostname', 'localhost')
port = int(self.config.get('port', '27017'))
database = self.config.get('database', None)
username = self.config.get('user', None)
password = self.config.get('password', None)
try:
if username and password and database:
uri = "mongodb://{}:{}@{}:{}/{}".format(
username, password, host, port, database)
cli = Client(uri)
check_db = cli[database]
check_db.authenticate(username, password)
return check_db.command("serverStatus")
elif username and password:
uri = "mongodb://{}:{}@{}:{}".format(username, password, host,
port)
cli = Client(uri)
return cli.test.command("serverStatus")
elif database:
uri = "mongodb://{}:{}/{}".format(host, port, database)
cli = Client(uri)
check_db = cli[database]
return check_db.command("serverStatus")
else:
cli = Client(host, port)
return cli.test.command("serverStatus")
except (ConnectionFailure, AutoReconnect):
self.exit(CRITICAL, message="unable to connect to mongodb")
def mongodb_stats(host, p, database, username, password):
port = int(p)
try:
if username and password and database:
c = Client("mongodb://"+username+":"+password+"@"+host+"/"+database, port)
elif username and password:
c = Client('mongodb://'+username+':'+password+'@'+host+'/', port)
elif database:
c = Client('mongodb://'+host+'/'+database, port)
else:
c = Client(host, port)
except ConnectionFailure, AutoReconnect:
return None
class MaccountHandler_for_URI(tornado.web.RequestHandler):
#连接MongoClient
client = Client() #为空则为默认设置 （'localhost',27017）
#连接数据库
db = client.traffic_project #如果存在则连接，不存在则创建
#连接聚集（collection） 相当于关系型数据库里的表
collection = db.maccount #如果存在连接，不存在创建
#get方法
def get(self, input):
if input == 'all':
resultset = MaccountHandler_for_URI.collection.find()
else:
resultset = MaccountHandler_for_URI.collection.find({}, {
input: 1,
'time': 1
})
result = resultset[resultset.count() - 1]
if not input == 'all' and not result.has_key(input):
print input + ' is not exist!'
self.write(u'路段id: "' + input + u'" 不存在')
else:
del result['_id']
jsonstr = json.dumps(result)
self.write(jsonstr)
def read_from_mongo(city):
client = Client(host='localhost', port=27017)
db = client['fs_db']
coll = db[city + '_good']
li = coll.find()
client.close()
return li
def write_to_mongo(ips):
client = Client(host='localhost', port=27017)
db = client['proxies_db']
coll = db['xici_proxies']
for ip in ips:
if coll.find({'ip': ip['ip']}).count() == 0:
coll.insert_one(ip)
client.close()
def getstate(timestr):
#连接MongoClient
client = Client() #为空则为默认设置 （'localhost',27017）
#连接数据库
db = client.traffic_project #如果存在则连接，不存在则创建
#由时间获得拥堵等级记录
result = db.state.find({'time': timestr})
return result[0]
def insert_traffic_state(traffic_state_dict):
#连接MongoClient
client = Client() #为空则为默认设置 （'localhost',27017）
#连接数据库
db = client.traffic_project #如果存在则连接，不存在则创建
#连接聚集（collection） 相当于关系型数据库里的表
traffic_state = db.traffic_state #如果存在连接，不存在创建
#插入数据
traffic_state.insert(traffic_state_dict)
def check_in_mongo(name):
client = Client(host=MONGO['uri'], port=MONGO['port'])
db = client['ipsite']
coll = db[name]
for i in coll.find():
i_result = check_ip_only(i['ip'], i['port'])
if not i_result:
print("delete i")
coll.delete_one(i)
def insert_congestion_index(congestion_index_dict):
#连接MongoClient
client = Client() #为空则为默认设置 （'localhost',27017）
#连接数据库
db = client.traffic_project #如果存在则连接，不存在则创建
#连接聚集（collection） 相当于关系型数据库里的表
congestion_index = db.congestion_index #如果存在连接，不存在创建
#插入数据
congestion_index.insert(congestion_index_dict)
def insert_maccount(maccount_dict):
#连接MongoClient
client = Client() #为空则为默认设置 （'localhost',27017）
#连接数据库
db = client.traffic_project #如果存在则连接，不存在则创建
#连接聚集（collection） 相当于关系型数据库里的表
maccount = db.maccount #如果存在连接，不存在创建
#插入数据
maccount.insert(maccount_dict)
def insert_speed(speed_dict):
#连接MongoClient
client = Client() #为空则为默认设置 （'localhost',27017）
#连接数据库
db = client.traffic_project #如果存在则连接，不存在则创建
#连接聚集（collection） 相当于关系型数据库里的表
speed = db.speed #如果存在连接，不存在创建
#插入数据
speed.insert(speed_dict)
def load_mongo(name):
ipsto = list()
client = Client(host=MONGO['uri'], port=MONGO['port'])
db = client["ipsite"]
col = db[name]
for x in col.find():
# print(x)
ipsto_ip = str(x['ip']) + ":" + str(x['port'])
ipsto.append(ipsto_ip)
return ipsto
def sto_mongo(array):
name = array[5]['type']
client = Client(host=MONGO['uri'], port=MONGO['port'])
db = client[MONGO['db']]
coll = db[name]
# 先检测，再写入，防止重复
for ip in array:
if coll.find({'ip': ip['ip']}).count() == 0:
coll.insert_one(ip)
client.close()
def write_to_mongo(ips):
'''将数据写入mongoDB'''
client = Client(host='localhost', port=27017)
db = client['proxies_db']
coll = db['proxies']
# 先检测，再写入，防止重复
for ip in ips:
if coll.find({'ip': ip['ip']}).count() == 0:
coll.insert_one(ip)
client.close()
def generate_ip(name):
mongo_array = list()
client = Client(MONGO['uri'], MONGO['port'])
db = client['ipsite']
coll = db[name]
# count = coll.count()
# nums = random.randint(0, count)
# print(nums)
mongo_ips = coll.find()
for i in mongo_ips:
mongo_ip = str(i['ip']) + ":" + str(i['port'])
mongo_array.append(mongo_ip)
return mongo_array
def clearmongodb():
client = Client()
db = client.traffic_project
#清空速度
db.speed.remove({})
#清空状态
db.state.remove({})
#清空流量
db.maccount.remove({})
#清空清空拥堵指数
db.congestion_index.remove({})
#清空其他指标数据
db.traffic_state.remove({})
def write_to_mongo(ips, city):
'''将数据写入mongoDB'''
client = Client(host='localhost', port=27017)
db = client['fs_db']
coll = db[city + '_good']
for ip in ips:
coll.insert_one({'name': ip[0], \
'price': ip[1],
'addresses': ip[2],
'areas': ip[3],
'eq': ip[4]})
client.close()
class SpeedHandler(tornado.web.RequestHandler):
#连接MongoClient
client = Client() #为空则为默认设置 （'localhost',27017）
#连接数据库
db = client.traffic_project #如果存在则连接，不存在则创建
#连接聚集（collection） 相当于关系型数据库里的表
collection = db.speed #如果存在连接，不存在创建
#post方法
def post(self):
segmentid = self.get_argument('segmentid', 'all')
callback = self.get_argument('callback')
print segmentid
if segmentid == 'all':
resultset = SpeedHandler.collection.find()
else:
resultset = SpeedHandler.collection.find({}, {
segmentid: 1,
'time': 1
})
result = resultset[resultset.count() - 1]
if not segmentid == 'all' and not result.has_key(segmentid):
print segmentid + ' is not exist!'
self.write(u'路段id: "' + segmentid + u'" 不存在')
else:
del result['_id']
jsonstr = json.dumps(result)
self.write(str(callback) + '(' + jsonstr + ');')
#get方法
def get(self):
segmentid = self.get_argument('segmentid', 'all')
callback = self.get_argument('callback', '')
print segmentid
if segmentid == 'all':
resultset = SpeedHandler.collection.find()
else:
resultset = SpeedHandler.collection.find({}, {
segmentid: 1,
'time': 1
})
result = resultset[resultset.count() - 1]
if not segmentid == 'all' and not result.has_key(segmentid):
print segmentid + ' is not exist!'
self.write(
str(callback) + '(' + u'路段id: "' + segmentid + u'" 不存在' + ');')
else:
del result['_id']
jsonstr = json.dumps(result)
self.write(callback + '([' + jsonstr + ']);')
def get_stats(self):
host = self.config.get('hostname', 'localhost')
port = int(self.config.get('port', '27017'))
username = self.config.get('user', None)
password = self.config.get('password', None)
try:
if username and password:
uri = "mongodb://{}:{}@{}:{}".format(username,
password,
host,
port)
cli = Client(uri)
admin_db = cli['admin']
admin_db.authenticate(username, password)
return admin_db.command("replSetGetStatus")
else:
cli = Client(host, port)
admin_db = cli['admin']
return admin_db.command("replSetGetStatus")
except (ConnectionFailure, AutoReconnect):
self.exit(CRITICAL, message="unable to connect to mongodb")
class CongestionRateHandler_for_URI(tornado.web.RequestHandler):
#连接MongoClient
client = Client() #为空则为默认设置 （'localhost',27017）
#连接数据库
db = client.traffic_project #如果存在则连接，不存在则创建
#连接聚集（collection） 相当于关系型数据库里的表
collection = db.traffic_state #如果存在连接，不存在创建
#get方法
def get(self):
resultset = CongestionRateHandler_for_URI.collection.find(
{}, {
'congrestion_rate': 1,
'time': 1
})
result = resultset[resultset.count() - 1]
del result['_id']
jsonstr = json.dumps(result)
self.write(jsonstr)
class MaccountHandler_for_WholeDay(tornado.web.RequestHandler):
#连接MongoClient
client = Client() #为空则为默认设置 （'localhost',27017）
#连接数据库
db = client.traffic_project #如果存在则连接，不存在则创建
#连接聚集（collection） 相当于关系型数据库里的表
collection = db.maccount #如果存在连接，不存在创建
#get方法
def get(self, input):
print input
resultset = MaccountHandler_for_WholeDay.collection.find(
{'time': re.compile(input)})
list = []
for result in resultset:
del result['_id']
list.append(result)
dict = {'history': list}
string = json.dumps(dict)
self.write(string)
def output(self):
if not self.output_params:
print(self.aims)
elif self.output_params.get('output_file'):
try:
print("Output to file" + self.output_params['output_file'])
result = {}
result[self.output_params['output_name']] = self.aims
json.dump(result, open(self.output_params['output_file'], 'w'))
except:
print("Error on json dump")
elif self.output_params.get('output_db'):
try:
db = Client(**self.output_params['output_db']).spyder
collection = db[self.output_params['output_name']]
collection.insert_many(self.aims)
except:
print("Error on db output")
else:
print("No valid output parameters found!")
#!/usr/bin/python
#coding:utf-8
import pymongo
from pymongo import MongoClient as Client
import datetime
import commands
import info
#create line
client = Client(info.commandClient)
#connect to database
db = client[info.commandDatabase]
#collections
menu = db[info.commandCollections]
#print db.collection_names()
# { "_id" : ObjectId("5986a816c8d00cd85382cc21"), "number" : "a", "command" : "db.serverStatus()" }
def insertCommand(filepath):
_dict = {}
try:
with open(filepath, 'r') as dict_file:
for line in dict_file:
if not line.startswith('#'):
if line.find(':') != -1:
(value, alias) = line.strip().split(':')
alias = int(alias) #the number need int to sort
while menu.find_one({'number': alias}) != None:
alias = alias + 1 ##make the number unique
new = {"Auto": "no", "number": alias, "command": value}
##the "Auto" is to diff the alise is defind or not,if defind,no
def mongodb_stats(host, port):
try:
c = Client(host, port)
except ConnectionFailure, AutoReconnect:
return None
#!/usr/bin/python
import argparse
from pymongo import MongoClient as Client
from bson import BSON
from bson import json_util
import json
import os
# mongo client
client = Client()
# script arguments
parser = argparse.ArgumentParser(
description='Export a Mongo Collection into json chunks given an offset')
parser.add_argument('--db',
metavar='d',
type=str,
help='specify a mongo db',
dest='db')
parser.add_argument('--collection',
metavar='c',
type=str,
help='specify a mongo collection',
dest='collection')
parser.add_argument('--offset',
metavar='O',
type=int,
help='specify an offset (default=1000)',
dest='offset')
parser.add_argument('--prefix',
#!/usr/bin/python
import argparse
from pymongo import MongoClient as Client
from bson import BSON
from bson import json_util
import json
import os
# mongo client
client = Client(
"mongodb://*****:*****@cityvision.documents.azure.com:10255/?ssl=true&replicaSet=globaldb"
)
# script arguments
parser = argparse.ArgumentParser()
parser.add_argument('--db',
metavar='d',
type=str,
help='specify a mongo db',
dest='db',
default='cityvision')
parser.add_argument('--collection',
metavar='c',
type=str,
help='specify a mongo collection',
dest='collection',
default='result')
parser.add_argument('--offset',
metavar='O',
type=int,
#!/usr/bin/python
#coding:utf-8
from pymongo import MongoClient as Client
import datetime
import info
#create line
client = Client(info.signInClient)
#connect to database
db = client[info.signInDatabase]
#collections
posts = db[info.signInCollections1]
glogs = db[info.signInCollections1]
perlogs = db.perOneDaily #save the each person's Msg number in one day one group
#print db.collection_names()
#print posts.find_one({"name":"pedro","group":"myGroup"})
# sign in num , return :your signIn number and total signIn number
def signIn(name, group):
string = posts.find_one({"name": name, "group": group})
# print "+++",string,"+++"
if string != None:
if string["date"] != datetime.datetime.now().strftime('%b-%d-%y'):
posts.update({
"name": name,
"group": group
}, {
"$set": {
"date": datetime.datetime.now().strftime('%b-%d-%y')
import chardet
import re
import ConfigParser
import hashlib
import cgi
apppath = os.path.dirname(os.path.abspath(__file__))
UPLOAD_FOLDER = apppath + '/pcapfiles/'
BUGS_FOLDER = apppath + '/bugsfiles/'
PER_FILE_PKTS = 100
ALLOWED_EXTENSIONS = set(['pcap','pcapng','cap'])
conn = None
db = None
count = 0
client = Client() #连接mongoclinet
mongodb = client.bugs #创建bugs数据库
bugs_info = mongodb.bugs_info #创建bugs_info聚集,相当于表
content_num = 0
mutex = threading.Lock()
#获取数据库连接
def get_connection():
global conn,db
if conn != None:
try:
conn.close()
except:
pass
conn = MySQLdb.connect(settings.DATABASES['default']['HOST'],\
import pytz
from bson.codec_options import CodecOptions
from pymongo import MongoClient as Client
from bson.json_util import dumps
from itchat.content import TEXT, PICTURE, VIDEO, RECORDING, ATTACHMENT, VIDEO
from wordcloud import WordCloud
import codecs
import jieba
import jieba.analyse
from scipy.misc import imread
import os
from os import path
import matplotlib.pyplot as plt
from PIL import Image, ImageDraw, ImageFont
connection = Client('mongodb://localhost:27017/')
mongodb = connection.get_database(
"wechat",
codec_options=CodecOptions(
tz_aware=True, tzinfo=pytz.timezone('Asia/Shanghai')))
msg_info = mongodb.msg_info
KEY = '684233fbc60b47d69f11a225b1d1c2ee'
def get_response(msg):
apiUrl = 'http://www.tuling123.com/openapi/api'
data = {
'key': KEY,
'info': msg,
#!/usr/bin/python
#coding:utf-8
from pymongo import MongoClient as Client
import datetime
#create line
client = Client('mongodb://*****:*****@127.0.0.1:27017/')
#connect to database
db = client.wechat
#collections
posts = db.signIn
glogs = db.GroupLogs
perlogs = db.perOneDaily #save the each person's Msg number in one day one group
#print db.collection_names()
#print posts.find_one({"name":"pedro","group":"myGroup"})
# sign in num , return :your signIn number and total signIn number
def signIn(name, group):
string = posts.find_one({"name": name, "group": group})
# print "+++",string,"+++"
if string != None:
if string["date"] != datetime.datetime.now().strftime('%b-%d-%y'):
posts.update({
"name": name,
"group": group
}, {
"$set": {
"date": datetime.datetime.now().strftime('%b-%d-%y')
}
