def __init__(self):
'''
init the common db connection transactions.
'''
self.client = mongo_client.MongoClient()
self.db = self.client.testnginxlog
self.data_user = None
def get_an_instance():
client = None
try:
client = mongo_client.MongoClient("mongodb://localhost:27017")
except Exception:
print("Database connection error.")
return client
def main():
# Parse the arguments
parser = argparse.ArgumentParser(
prog='surveil-pack-upload',
add_help=False,
)
parser.add_argument('--mongo-uri',
default='mongodb://localhost:27017',
help='Defaults to localhost',
type=str)
parser.add_argument('pack',
metavar='[Pack]',
type=str,
nargs=1,
help='Pack directory')
(options, args) = parser.parse_known_args(sys.argv[1:])
pack_dir = options.pack[0]
pack_name = os.path.basename(os.path.normpath(pack_dir))
# Find the .cfg files
cfg_files = [
os.path.join(dirpath, f)
for dirpath, dirnames, files in os.walk(pack_dir)
for f in fnmatch.filter(files, '*.cfg')
]
# Load the config
conf = config.Config()
loaded_conf = conf.read_config(cfg_files)
raw_objects = conf.read_config_buf(loaded_conf)
# Remove the empty items
non_empty_config = {k: v for k, v in raw_objects.items() if v}
for config_type in non_empty_config:
for config_item in non_empty_config[config_type]:
# Tag the config objects
config_item['SURVEIL_PACK_NAME'] = pack_name
# Replace lists with csv
items_to_modify = ([
i for i in config_item.items() if isinstance(i[1], list)
])
for i in items_to_modify:
config_item[i[0]] = ','.join(i[1])
# Remove the existing pack from mongodb
mongo = mongo_client.MongoClient(options.mongo_uri)
mongo_shinken = mongo.shinken
for collection in ([
c for c in mongo_shinken.collection_names()
if not c.startswith("system.")
]):
mongo_shinken[collection].remove({'SURVEIL_PACK_NAME': pack_name})
# Add the replacement pack
for config_type in non_empty_config:
mongo_shinken[config_type + 's'].insert(non_empty_config[config_type])
def connect(self, uri, database, collection=None):
"""
Positional Arguments:
- uri (string) - Mongo-compliant connection uri string
database (string) - name of the database you want to connect to
Optional Arguments:
- collection (string) - name of the collection you want to reach
(Default: None)
"""
try:
self._client = mongo_client.MongoClient(uri)
self._database = self._client[database]
# Reaching collection names to check whether the received exists
print 'Database Status : %s' % self._database.last_status()
if collection:
self._collection = self._database[collection]
return True
except errors.OperationFailure as e:
print 'Error Code : %s - Details: %s' % (e.code, e.details)
return False
def __init__(self, client=None, disk_cache=None, db_name="video_cache", collection_name='default'):
uri = "mongodb://%s:%s@%s" % (
quote_plus(configs.MONGO_USERNAME), quote_plus(configs.MONGO_PASSWORD), configs.MONGO_HOST)
self.client = (mongo_client.MongoClient(uri) if not client else client
)
self.db = self.client[db_name]
self.collection = self.db[collection_name]
self.disk_cache = disk_cache
def test_connection_and_basic_query(self):
client = mongo_client.MongoClient(
'mongodb://ds031651.mongolab.com:31651')
database = client['vppdev']
database.authenticate('vppdev', 'vpp2015', mechanism='SCRAM-SHA-1')
collection = database['registro']
document = collection.find_one()
client.close()
print(type(document))
def get_an_instance():
client = None
try:
client = mongo_client.MongoClient(
"mongodb://*****:*****@localhost:27017/lifewatcher?authMechanism=SCRAM-SHA-1"
)
except Exception:
print("Database connection error.")
return client
def __init__(self):
self.mongodb_host=settings.MONGODB_HOST
self.mongodb_port = settings.MONGODB_PORT
self.mongodb_db = settings.MONGODB_DB
self.mongodb_password = settings.MONGODB_PASSWORD
self.mongodb_username = settings.MONGODB_USERNAME
self.client = mongo_client.MongoClient(
"mongodb://" + self.mongodb_username + ":" + self.mongodb_password + "@" + self.mongodb_host + ":"
+ self.mongodb_port + "/" + self.mongodb_db)
def __init__(self,
path='./policy.csv',
host='127.0.0.1',
port=27017,
db_name='szkj',
doc_name='policy'):
self.to_path = path
self.conn = mongo_client.MongoClient(host=host, port=port)
self.db = self.conn[db_name]
self.doc = self.db[doc_name]
if not self.conn:
raise Exception('Connection failure')
def populate_user_data():
"""
Pre-populate user data for the app, including an admin account
"""
try:
db = mongo_client.MongoClient(config.MONGO_URI).twitter
db.user.insert_one({
'username': '******',
'password': '******',
})
print("Created an admin account")
except Exception as e:
print(e)
def test_insertion(self):
client = mongo_client.MongoClient(
'mongodb://ds031651.mongolab.com:31651')
database = client['vppdev']
database.authenticate('vppdev', 'vpp2015', mechanism='SCRAM-SHA-1')
collection = database['registro']
new_document = {
u'Port': 25,
u'Server': u'localhost',
u'api': u'custom'
}
print(type(new_document))
collection.insert(new_document)
print(dumps(new_document))
def __init__(self, db_name, host_name=settings.hostname):
"""
init the common db connection transactions.
"""
self.client = mongo_client.MongoClient(host=host_name)
self.db = self.client.__getattr__(db_name)
#init all the collections
#set attr self.colname = db.col
self.col_name_dict = DBCol().__dict__
for i in range(len(self.col_name_dict)):
self.__setattr__(
self.col_name_dict.keys()[i],
self.db.__getattr__(self.col_name_dict.values()[i]))
def set_config(host,port,db,user,password):
global _coll_language
global _db
global _coll_error
from pymongo import mongo_client
cnn = mongo_client.MongoClient(host=host, port=port)
_db = cnn.get_database(db)
_db.authenticate(user, password)
_coll_language = _db.get_collection("sys.export_import_language")
_coll_error = _db.get_collection("sys.export_import_error")
pass
def __init__(self, conn_cred):
"""
Spins up the MongoDB Cluster with the credentials needed
@:conn_cred, an XML parsed object containing necessary attributes
"""
self.loggerManager = DBLoggerManager()
self.issues_logger = self.loggerManager.getIssuesLogger()
self.queries_logger = self.loggerManager.getQueriesLogger()
try:
self.client = mongo_client.MongoClient(conn_cred)
self.queries_logger.log(logging.INFO, "Connected!")
print("Connected")
except mongo_client.InvalidURI as e:
self.issues_logger.log(
logging.CRITICAL,
"Connection failed. Info {0}".format(e.message))
def getConn(self):
rootLogger.debug("MongoDaoSupport getConn start")
conn = mongo_client.MongoClient(
host=MongoConfig.HOST,
port=MongoConfig.PORT,
document_class=MongoConfig.DOCUMENT_CLASS,
tz_aware=MongoConfig.TZ_AWARE,
connect=MongoConfig.CONNECT,
maxPoolSize=MongoConfig.MAX_POOL_SIZE,
minPoolSize=MongoConfig.MIN_POOL_SIZE
)
# 建立和数据库系统的连接,创建Connection时，指定host及port参数
db_auth = conn.admin
# admin 数据库有帐号，连接-认证-切换库
db_auth.authenticate(MongoConfig.USER, MongoConfig.PWD)
try:
conn.admin.command('ismaster')
except ConnectionFailure:
print("Server not available")
rootLogger.debug("MongoDaoSupport getConn end")
return conn
def populate_hateword_data():
"""
Populate the `hateword` table in MongoDB with data from CSV file.
"""
with open("./data/hate-speech-lexicons/refined_ngram_dict.csv") as f:
lst = [row.split(',', 1)[0] for row in f]
lst = lst[1:]
lst = [{
'word': word,
'category': [],
'similar_to': []
} for word in lst]
try:
db = mongo_client.MongoClient(config.MONGO_URI).twitter
db.hateword.delete_many({})
result = db.hateword.insert_many(lst)
print("Completed populating", len(result.inserted_ids),
"hate words")
except pymongo.errors.BulkWriteError as e:
print(e.details)
# and then get the location info from ip2location.com ,store them into the mongodb.
from mongobase import Conn
__author__ = 'fandongyun'
from utils import IpLocation
from utils import IpLocationFetch
from mongobase.Settings import DBColNames
from pymongo import mongo_client
from requests import ConnectionError
log_conn = Conn("NginxEventLog")
log_db = log_conn.get_db()
client = mongo_client.MongoClient(host="172.26.178.208")
ip_db = client.__getattr__("IPLocationDB")
ip_db_col_name = "IPLocationInfo"
def find_ip_from_collection(collection_name):
distinct_ip = log_db.__getattr__(collection_name).distinct("remote_addr")
print len(distinct_ip)
return distinct_ip
def is_ip_resolved(ip):
#TODO
ip_count = ip_db.__getattr__(ip_db_col_name).find({"ip": ip}).count()
if ip_count > 0:
return True
import matplotlib.pyplot as plt from pymongo import mongo_client import pprint import time uri = "mongodb+srv://github_reader:[email protected]/admin?ssl=true&authSource=admin&retryWrites=true&w=majority" client = mongo_client.MongoClient( "mongodb+srv://github_reader:[email protected]/admin?ssl=true&authSource=admin&retryWrites=true&w=majority" ) db = client['cryptomood'] collection = db.get_collection('github') symbols = set() print("1") for document in collection.find(): if document["symbol"] not in symbols: symbols.add(document["symbol"]) sym = document["symbol"] print(sym)
def __init__(self):
client = mongo_client.MongoClient(host=db_host, port=db_port)
self.rows = client['logging']['rows']
self.rows.ensure_index('session_id', ASCENDING, unique=True)
self.rows.ensure_index('leased_at', ASCENDING)
import gridfs
from pymongo import mongo_client
from PIL import Image
c = mongo_client.MongoClient()
db = c.botPhotos
fs = gridfs.GridFS(db)
def save_image(path, c_id, num):
fs.put(path.encode('ascii'), chat=c_id, filename=f"{c_id}_{num}.jpg")
from io import BytesIO
import requests
def load_image(c_id):
photos = []
for grid_out in fs.find({"chat": c_id}):
file = grid_out.read()
a = requests.get(file.decode("utf-8", "ignore"))
img = Image.open(BytesIO(a.content))
photos.append(img)
return photos
# directory = r"H:\temp\Ebeam_RunLog"
import re
import os
import traceback
import sys
from pprint import pprint
import datetime
from dateutil.parser import parse
# open database connections
from pymongo import mongo_client
from pprint import pprint
client = mongo_client.MongoClient('mongodb://tali-e7440z:27017/')
db = client['result_database'] # or db = client.test_database
collection = db['fileStor'] # or collection = db.posts
import pickle
# from os import walk
from scandir import \
walk # use scandir.walk instead of os.walk - its faster (5-10x faster on win64; 3x on Linux, MacOS)
deepscanflag = True
# DEEP SCAN: All dir and subdirs needed for user folders K Drive etc.
def listFilesWithPattern2(
directory, extension): # working version for one type of extension
files_set = set()
def main():
c = mongo_client.MongoClient(connectTimeoutMS=60 * 1000) # jack up timeout
c.drop_database("benchmark")
db = c.benchmark
timed("insert (small, no index)", insert, [db, 'small_none', small],
setup_insert)
timed("insert (medium, no index)", insert, [db, 'medium_none', medium],
setup_insert)
timed("insert (large, no index)", insert, [db, 'large_none', large],
setup_insert)
db.small_index.create_index("x", ASCENDING)
timed("insert (small, indexed)", insert, [db, 'small_index', small])
db.medium_index.create_index("x", ASCENDING)
timed("insert (medium, indexed)", insert, [db, 'medium_index', medium])
db.large_index.create_index("x", ASCENDING)
timed("insert (large, indexed)", insert, [db, 'large_index', large])
timed("batch insert (small, no index)", insert_batch,
[db, 'small_bulk', small], setup_insert)
timed("batch insert (medium, no index)", insert_batch,
[db, 'medium_bulk', medium], setup_insert)
timed("batch insert (large, no index)", insert_batch,
[db, 'large_bulk', large], setup_insert)
timed("find_one (small, no index)", find_one,
[db, 'small_none', per_trial / 2])
timed("find_one (medium, no index)", find_one,
[db, 'medium_none', per_trial / 2])
timed("find_one (large, no index)", find_one,
[db, 'large_none', per_trial / 2])
timed("find_one (small, indexed)", find_one,
[db, 'small_index', per_trial / 2])
timed("find_one (medium, indexed)", find_one,
[db, 'medium_index', per_trial / 2])
timed("find_one (large, indexed)", find_one,
[db, 'large_index', per_trial / 2])
timed("find (small, no index)", find, [db, 'small_none', per_trial / 2])
timed("find (medium, no index)", find, [db, 'medium_none', per_trial / 2])
timed("find (large, no index)", find, [db, 'large_none', per_trial / 2])
timed("find (small, indexed)", find, [db, 'small_index', per_trial / 2])
timed("find (medium, indexed)", find, [db, 'medium_index', per_trial / 2])
timed("find (large, indexed)", find, [db, 'large_index', per_trial / 2])
# timed("find range (small, no index)", find,
# [db, 'small_none',
# {"$gt": per_trial / 4, "$lt": 3 * per_trial / 4}])
# timed("find range (medium, no index)", find,
# [db, 'medium_none',
# {"$gt": per_trial / 4, "$lt": 3 * per_trial / 4}])
# timed("find range (large, no index)", find,
# [db, 'large_none',
# {"$gt": per_trial / 4, "$lt": 3 * per_trial / 4}])
timed("find range (small, indexed)", find, [
db, 'small_index', {
"$gt": per_trial / 2,
"$lt": per_trial / 2 + batch_size
}
])
timed("find range (medium, indexed)", find, [
db, 'medium_index', {
"$gt": per_trial / 2,
"$lt": per_trial / 2 + batch_size
}
])
timed("find range (large, indexed)", find, [
db, 'large_index', {
"$gt": per_trial / 2,
"$lt": per_trial / 2 + batch_size
}
])
def configure_and_connect(collection_name):
client = mongo_client.MongoClient('mongodb://ds031651.mongolab.com:31651')
database = client['vppdev']
database.authenticate('vppdev', 'vpp2015', mechanism='SCRAM-SHA-1')
collection = database[collection_name]
return {'client': client, 'collection': collection}
def __init__(self):
self.logger = getLogger("LotteryDatabase")
self.client = mongo_client.MongoClient(environ["MONGO_URI"])
self.lottery = self.client['bepis_bot']['lottery']
def __init__(self):
self.logger = getLogger("CodeDatabase")
self.client = mongo_client.MongoClient(environ["MONGO_URI"])
self.codes = self.client['bepis_bot']['codes']
self.codes.create_index("hash", unique=True)
def __init__(self):
self.logger = getLogger("InviteDatabase")
self.client = mongo_client.MongoClient(environ["MONGO_URI"])
self.invites = self.client['bepis_bot']['invites']
self.profiles = self.client['bepis_bot']['profiles']
super().__init__()
"""Handles database connection, and all that fun stuff. Adds a wrapper to pymongo. @package ppbot """ from pymongo import mongo_client from settings import * client = mongo_client.MongoClient(MONGO_HOST, MONGO_PORT) db = client[MONGO_DB]
post = {
"Keywords": categories
}
posts = db["category_words"]
posts.insert_one(post)
pprint.pprint(posts.find())
print('Success!')
print('(Make sure mongod is running...)')
time.sleep(1)
labels = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sept', 'Oct', 'Nov', 'Dec']
year = '2018'
# year = datetime.datetime.today().strftime('%Y')
# month = datetime.datetime.today().strftime('%m')
wb = px.load_workbook('{} Monthly Expenses.xlsx'.format(year))
month = input('what month?\n')
month_data = wb.get_sheet_by_name(labels[int(month)-1])
month_max = month_data.max_row
client = mongo_client.MongoClient('localhost', 27017)
db = client.month_expenses
# update_expenses()
category_upload()
def __init__(self, name: str):
self.logger = getLogger(name + "-database")
self.client = mongo_client.MongoClient(environ["MONGO_URI"])
self.profiles = self.client['bepis_bot']['profiles']
self.profiles.create_index("user_id", unique=True)
super().__init__()
from pymongo import mongo_client
import binascii
BITS_MTU = 1522 * 8
HEX_MTU = 1522 * 2
mc = mongo_client.MongoClient("localhost", 27017)
collection = mc["Packet_Snoop"].collection.Raw_Packet
def build_cardinal(cardinal, y):
c = cardinal.get(y)
if c:
return c
else:
id = len(cardinal.keys()) + 1
cardinal[y] = id
return id
def write_row_to_result_file(fh, doc, cardinal, count, raw=False):
y = doc["Y"]
l = doc["Length"]
x = binascii.hexlify(doc["packet"]).decode('utf-8')
if raw:
row = [str(int(i, 16)) for i in x]
else:
row = [str(round(int(i, 16) / 15, 2)) for i in x]
